diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..94bf9acc --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,330 @@ +# KoInsight Development Guide + +This guide covers everything you need to know to develop KoInsight locally. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Getting Started](#getting-started) +- [Development Workflow](#development-workflow) +- [Key Technologies](#key-technologies) +- [Project Structure](#project-structure) +- [Database Management](#database-management) +- [Code Quality](#code-quality) +- [Testing](#testing) +- [Contributing](#contributing) + +## Prerequisites + +### Required Dependencies + +Before you begin, ensure you have the following installed: + +1. **Node.js** (v22 or higher) + - Download from [nodejs.org](https://nodejs.org/) + - Or use a version manager like [nvm](https://github.com/nvm-sh/nvm) + +2. **npm** (v10.2.4 or higher) + - The project uses npm workspaces for monorepo management + +### Recommended Tools + +- **nvm** (Node Version Manager) - Makes it easy to switch between Node versions + ```bash + # Install nvm (macOS/Linux) + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash + + # Use the required Node version + nvm install 22 + nvm use 22 + ``` + +- **Docker** (optional) - For running the production build locally + +## Getting Started + +### 1. Clone the Repository + +```bash +git clone https://github.com/GeorgeSG/koinsight.git +cd koinsight +``` + +### 2. Install Dependencies + +KoInsight uses a monorepo structure with npm workspaces. Install all dependencies from the root: + +```bash +npm install +``` + +This will install dependencies for: +- Root workspace (build tools, Prettier, Turbo) +- `apps/server` (Express backend) +- `apps/web` (React frontend) +- `packages/common` (shared types) + +### 3. Set Up the Database + +The development database uses SQLite and is stored in the `data/` directory. + +Run database migrations: + +```bash +npm run -w server knex migrate:latest +``` + +### 4. Seed the Database (Optional but Recommended) + +Populate your database with realistic fake data for development: + +```bash +# From the root directory +npm run seed +``` + +This creates: +- 5 e-reader devices +- 10 books (popular fantasy/sci-fi titles) +- Realistic reading statistics and page data +- 200+ annotations (highlights, notes, bookmarks) +- 14 genres with book associations +- 3 test users with KoSync progress data + +**Test User Credentials:** +- Username: `reader1`, `reader2`, `bookworm` +- Password: `password123` (all users) + +See [Database Seeding](#database-seeding) for more details. + +## Development Workflow + +### Running the Development Server + +KoInsight consists of two apps that run concurrently: + +#### Option 1: Run Both Apps Together (Recommended) + +From the **root directory**: + +```bash +npm run dev +``` + +This uses Turbo to run both apps in parallel: +- **Backend server**: http://localhost:3000 (Express API) +- **Frontend web app**: http://localhost:5173 (Vite dev server) + +#### Option 2: Run Apps Individually + +**Backend only:** +```bash +cd apps/server +npm run dev +``` +- Runs on http://localhost:3001 +- Watches for TypeScript changes and auto-restarts + +**Frontend only:** +```bash +cd apps/web +npm run dev +``` +- Runs on http://localhost:3000 +- Hot module replacement enabled + + +### Development Tips + +1. **Frontend proxy**: The Vite dev server (port 3000) proxies API requests to the backend (port 3001) +2. **Hot reload**: Both apps support hot reloading during development +3. **TypeScript**: Changes to TypeScript files trigger automatic recompilation +4. **Shared types**: The `@koinsight/common` package contains types shared between frontend and backend + + +## Key Technologies + +**Backend:** +- Express 5.x - Web framework +- Knex.js - SQL query builder +- better-sqlite3 - SQLite driver +- bcryptjs - Password hashing +- Multer - File upload handling +- Zod - Schema validation + +**Frontend:** +- React 18.x - UI library +- Vite - Build tool and dev server +- Mantine UI - Component library +- React Router 7.x - Client-side routing +- SWR - Data fetching and caching +- Recharts - Data visualization + +**Development:** +- TypeScript - Type safety +- Turbo - Monorepo build system +- Prettier - Code formatting +- Vitest - Unit testing + + +## Project Structure + +``` +koinsight/ +├── apps/ +│ ├── server/ # Express backend (TypeScript) +│ │ ├── src/ +│ │ │ ├── annotations/ # Annotation management +│ │ │ ├── books/ # Book management +│ │ │ ├── db/ # Database migrations, seeds, factories +│ │ │ ├── devices/ # Device management +│ │ │ ├── genres/ # Genre management +│ │ │ ├── kosync/ # KoSync protocol implementation +│ │ │ ├── stats/ # Statistics and analytics +│ │ │ └── app.ts # Express app entry point +│ │ └── package.json +│ └── web/ # React frontend (Vite + TypeScript) +│ ├── src/ +│ │ ├── components/ # React components +│ │ ├── pages/ # Page components +│ │ ├── api/ # API client functions +│ │ └── main.tsx # App entry point +│ └── package.json +├── packages/ +│ └── common/ # Shared types and utilities +│ └── types/ # TypeScript type definitions +├── data/ # SQLite database files (gitignored) +├── package.json # Root workspace config +├── turbo.json # Turbo build configuration +└── .prettierrc # Prettier configuration +``` + +## Database Management + +### Database Overview + +- **Engine**: SQLite (via better-sqlite3) +- **Query Builder**: Knex.js +- **Location**: `data/dev.db` (development), `data/prod.db` (production) +- **Migrations**: Located in `apps/server/src/db/migrations/` +- **Seeds**: Located in `apps/server/src/db/seeds/` + +### Running Migrations + +```bash +# Run all pending migrations +npm run -w server knex migrate:latest + +# Rollback last migration +npm run -w serverknex migrate:rollback + +# Create a new migration +npm run -w server knex migrate:make migration_name +``` + +### Database Seeding + +Seed the database with realistic fake data: + +```bash +# From root directory +npm run seed +``` + +**What gets seeded:** + +| Data Type | Count | Description | +|-----------|-------|-------------| +| Devices | 5 | Kindle, Kobo, Nook, iPad, Android Tablet | +| Books | 10 | Popular fantasy/sci-fi titles | +| Book-Device Associations | 50 | Each book on each device | +| Page Statistics | ~1,800 | Reading progress over last 100 days | +| Annotations | ~200 | Highlights, notes, and bookmarks | +| Genres | 14 | Fantasy, Sci-Fi, etc. with book associations | +| Users | 3 | Test accounts (password: `password123`) | +| Progress Records | ~13 | KoSync reading progress | + + +### Advanced Knex Commands + +```bash +# Run a specific seed file +npm run -w server knex seed:run -- --specific=01_devices.ts + +# Create a new seed file +npm run -w server knex seed:make new_seed_name + +# View migration status +npm run -w server knex migrate:status +``` + +### Resetting the Database + +If you need a fresh start: + +```bash +# Delete the database +rm data/dev.db + +# Run migrations +npm run -w server knex migrate:latest + +# Seed with fake data +npm run seed +``` + +## Code Quality + +### Code Formatting + +KoInsight uses [Prettier](https://prettier.io/) for consistent code formatting. + +**Prettier Configuration** (`.prettierrc`): + +**Format your code:** + +```bash +# Format all files (from root) +npx prettier --write . + +# Format specific files +npx prettier --write "apps/server/**/*.ts" +npx prettier --write "apps/web/**/*.{ts,tsx}" + +# Check formatting without changing files +npx prettier --check . +``` + +**Editor Integration:** +- [VS Code Extension](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) +- Enable "Format on Save" for automatic formatting + + +## Testing + +### Running Tests + +```bash +# Run all tests +npm run test:coverage + +# Run server tests only +cd apps/server +npm test + +# Run tests in watch mode +npm run test:watch + +# Run with coverage report +npm run test:coverage +``` + +## Contributing + +When contributing code: + +1. **Format your code** with Prettier before committing +2. **Run tests** to ensure nothing breaks +3. **Write tests** for new features +4. **Update documentation** if needed +5. **Follow existing patterns** in the codebase diff --git a/README.md b/README.md index 900914a5..9a2bb915 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,9 @@ You can use your KoInsight instance as a KOReader sync server. This allows you t The progress sync data should appear in the **"Progress syncs"** page in KoInsight. +# Development +See [DEVELOPMENT.md](DEVELOPMENT.md) for development setup and instructions. + # Roadmap (a.k.a things I want to do) diff --git a/apps/server/src/db/factories/progress-factory.ts b/apps/server/src/db/factories/progress-factory.ts new file mode 100644 index 00000000..89559bb3 --- /dev/null +++ b/apps/server/src/db/factories/progress-factory.ts @@ -0,0 +1,45 @@ +import { Progress, User, Device } from '@koinsight/common/types'; +import { faker } from '@faker-js/faker'; +import { Knex } from 'knex'; + +type FakeProgress = Omit; + +export function fakeProgress( + user: User, + device: Device, + documentName: string, + overrides: Partial = {} +): FakeProgress { + const percentage = faker.number.float({ min: 0, max: 100, fractionDigits: 2 }); + + const progress: FakeProgress = { + user_id: user.id!, + document: documentName, + progress: JSON.stringify({ + page: faker.number.int({ min: 1, max: 500 }), + position: faker.number.float({ min: 0, max: 1, fractionDigits: 4 }), + }), + percentage, + device: device.model, + device_id: device.id, + ...overrides, + }; + + return progress; +} + +export async function createProgress( + db: Knex, + user: User, + device: Device, + documentName: string, + overrides: Partial = {} +): Promise { + const progressData = fakeProgress(user, device, documentName, overrides); + const date = new Date(); + const [progress] = await db('progress') + .insert({ ...progressData, created_at: date, updated_at: date }) + .returning('*'); + + return progress; +} diff --git a/apps/server/src/db/factories/user-factory.ts b/apps/server/src/db/factories/user-factory.ts new file mode 100644 index 00000000..46392f6d --- /dev/null +++ b/apps/server/src/db/factories/user-factory.ts @@ -0,0 +1,30 @@ +import { User } from '@koinsight/common/types'; +import { faker } from '@faker-js/faker'; +import bcrypt from 'bcryptjs'; +import { Knex } from 'knex'; + +const SALT_ROUNDS = 12; + +type FakeUser = Omit; + +export async function fakeUser(overrides: Partial = {}): Promise { + const password = overrides.password || 'password123'; + const password_hash = await bcrypt.hash(password, SALT_ROUNDS); + + const user: FakeUser = { + username: overrides.username || faker.internet.username(), + password_hash, + }; + + return user; +} + +export async function createUser( + db: Knex, + overrides: Partial = {} +): Promise { + const userData = await fakeUser(overrides); + const [user] = await db('user').insert(userData).returning('*'); + + return user; +} diff --git a/apps/server/src/db/seeds/01_devices.ts b/apps/server/src/db/seeds/01_devices.ts index a50b02d2..66ed90a8 100644 --- a/apps/server/src/db/seeds/01_devices.ts +++ b/apps/server/src/db/seeds/01_devices.ts @@ -20,4 +20,5 @@ export async function seed(knex: Knex): Promise { const devices = await Promise.all(SEED_DEVICES.map((device) => createDevice(db, device))); SEEDED_DEVICES = devices as Device[]; + console.log(`✓ Seeded ${SEEDED_DEVICES.length} devices`); } diff --git a/apps/server/src/db/seeds/02_books.ts b/apps/server/src/db/seeds/02_books.ts index b681837e..9131d0a9 100644 --- a/apps/server/src/db/seeds/02_books.ts +++ b/apps/server/src/db/seeds/02_books.ts @@ -94,4 +94,5 @@ export async function seed(knex: Knex): Promise { const books = await Promise.all(SEED_BOOKS.map((book) => createBook(db, book))); SEEDED_BOOKS = books as Book[]; + console.log(`✓ Seeded ${SEEDED_BOOKS.length} books`); } diff --git a/apps/server/src/db/seeds/03_book_devices.ts b/apps/server/src/db/seeds/03_book_devices.ts index 918136fe..282a0b12 100644 --- a/apps/server/src/db/seeds/03_book_devices.ts +++ b/apps/server/src/db/seeds/03_book_devices.ts @@ -19,4 +19,5 @@ export async function seed(knex: Knex): Promise { }); SEEDED_BOOK_DEVICES = await Promise.all(promises); + console.log(`✓ Seeded ${SEEDED_BOOK_DEVICES.length} book-device associations`); } diff --git a/apps/server/src/db/seeds/04_page_stats.ts b/apps/server/src/db/seeds/04_page_stats.ts index 516fdedb..6da842b5 100644 --- a/apps/server/src/db/seeds/04_page_stats.ts +++ b/apps/server/src/db/seeds/04_page_stats.ts @@ -58,5 +58,6 @@ export async function seed(knex: Knex): Promise { } }); - await Promise.all(promises); + const pageStats = await Promise.all(promises); + console.log(`✓ Seeded ${pageStats.length} page statistics`); } diff --git a/apps/server/src/db/seeds/05_annotations.ts b/apps/server/src/db/seeds/05_annotations.ts new file mode 100644 index 00000000..e43beae4 --- /dev/null +++ b/apps/server/src/db/seeds/05_annotations.ts @@ -0,0 +1,126 @@ +import { Annotation, AnnotationType, Book, Device } from '@koinsight/common/types'; +import { subDays, subHours } from 'date-fns'; +import { Knex } from 'knex'; +import { db } from '../../knex'; +import { createAnnotation } from '../factories/annotation-factory'; +import { SEEDED_BOOKS } from './02_books'; +import { SEEDED_DEVICES } from './01_devices'; + +/** + * Generate realistic annotations for a book + * - 5-15 highlights per book + * - 2-5 notes per book + * - 3-8 bookmarks per book + */ +function generateAnnotationsForBook(book: Book, device: Device): Promise[] { + const promises: Promise[] = []; + const today = new Date(); + + // Determine how many pages have been read based on reference_pages + const totalPages = book.reference_pages ?? 300; + const readProgress = Math.random() * 0.8 + 0.2; // 20-100% read + const maxPage = Math.floor(totalPages * readProgress); + + // Track used page/datetime combinations to avoid duplicates + const usedCombinations = new Set(); + + // Helper to generate unique datetime for a page + const getUniqueDateTime = (pageno: number, daysAgo: number): string => { + let datetime = subDays(today, daysAgo).toISOString(); + let key = `${pageno}-${datetime}`; + let attempts = 0; + + // If this combination exists, add some hours to make it unique + while (usedCombinations.has(key) && attempts < 100) { + datetime = subHours(new Date(datetime), Math.floor(Math.random() * 24) + 1).toISOString(); + key = `${pageno}-${datetime}`; + attempts++; + } + + usedCombinations.add(key); + return datetime; + }; + + // Generate highlights (5-15 per book) + const numHighlights = Math.floor(Math.random() * 11) + 5; + for (let i = 0; i < numHighlights; i++) { + const daysAgo = Math.floor(Math.random() * 30); // Within last 30 days + const pageno = Math.floor(Math.random() * maxPage) + 1; + const datetime = getUniqueDateTime(pageno, daysAgo); + + promises.push( + createAnnotation(db, book, device, 'highlight', { + pageno, + page_ref: String(pageno), + datetime, + datetime_updated: subDays(new Date(datetime), -1).toISOString(), + chapter: `Chapter ${Math.floor(pageno / 20) + 1}`, + total_pages: totalPages, + drawer: ['lighten', 'underscore', 'invert'][Math.floor(Math.random() * 3)] as any, + color: ['red', 'orange', 'yellow', 'green', 'olive', 'cyan', 'blue', 'purple', 'gray'][ + Math.floor(Math.random() * 4) + ] as any, + }) + ); + } + + // Generate notes (2-5 per book) + const numNotes = Math.floor(Math.random() * 4) + 2; + for (let i = 0; i < numNotes; i++) { + const daysAgo = Math.floor(Math.random() * 30); + const pageno = Math.floor(Math.random() * maxPage) + 1; + const datetime = getUniqueDateTime(pageno, daysAgo); + + promises.push( + createAnnotation(db, book, device, 'note', { + pageno, + page_ref: String(pageno), + datetime, + datetime_updated: subDays(new Date(datetime), -1).toISOString(), + chapter: `Chapter ${Math.floor(pageno / 20) + 1}`, + total_pages: totalPages, + color: ['yellow', 'red', 'blue', 'green'][Math.floor(Math.random() * 4)] as any, + }) + ); + } + + // Generate bookmarks (3-8 per book) + const numBookmarks = Math.floor(Math.random() * 6) + 3; + for (let i = 0; i < numBookmarks; i++) { + const daysAgo = Math.floor(Math.random() * 30); + const pageno = Math.floor(Math.random() * maxPage) + 1; + const datetime = getUniqueDateTime(pageno, daysAgo); + + promises.push( + createAnnotation(db, book, device, 'bookmark', { + pageno, + page_ref: String(pageno), + datetime, + datetime_updated: subDays(new Date(datetime), -1).toISOString(), + chapter: `Chapter ${Math.floor(pageno / 20) + 1}`, + total_pages: totalPages, + }) + ); + } + + return promises; +} + +export let SEEDED_ANNOTATIONS: Annotation[] = []; + +export async function seed(knex: Knex): Promise { + await knex('annotation').del(); + + const promises: Promise[] = []; + + // Generate annotations for each book + SEEDED_BOOKS.forEach((book) => { + // Use a random device for each book (simulating different reading devices) + const device = SEEDED_DEVICES[Math.floor(Math.random() * SEEDED_DEVICES.length)]; + promises.push(...generateAnnotationsForBook(book, device)); + }); + + SEEDED_ANNOTATIONS = await Promise.all(promises); + + console.log(`✓ Seeded ${SEEDED_ANNOTATIONS.length} annotations`); +} diff --git a/apps/server/src/db/seeds/06_genres.ts b/apps/server/src/db/seeds/06_genres.ts new file mode 100644 index 00000000..12d7a362 --- /dev/null +++ b/apps/server/src/db/seeds/06_genres.ts @@ -0,0 +1,78 @@ +import { Genre } from '@koinsight/common/types'; +import { Knex } from 'knex'; +import { db } from '../../knex'; +import { createGenre } from '../factories/genre-factory'; +import { SEEDED_BOOKS } from './02_books'; + +// Define realistic genres for books +const GENRES = [ + 'Fantasy', + 'Science Fiction', + 'Epic Fantasy', + 'Urban Fantasy', + 'Space Opera', + 'Hard Science Fiction', + 'Adventure', + 'Magic', + 'Military Fiction', + 'Post-Apocalyptic', + 'Time Travel', + 'Dystopian', + 'Cyberpunk', + 'Sword and Sorcery', +]; + +// Map books to their genres (by title pattern matching) +const BOOK_GENRE_MAPPING: { [key: string]: string[] } = { + 'Mistborn': ['Fantasy', 'Epic Fantasy', 'Magic', 'Adventure'], + 'The Name of the Wind': ['Fantasy', 'Adventure', 'Magic'], + 'A Game of Thrones': ['Fantasy', 'Epic Fantasy', 'Adventure', 'Military Fiction'], + 'The Way of Kings': ['Fantasy', 'Epic Fantasy', 'Adventure'], + 'The Fellowship of the Ring': ['Fantasy', 'Epic Fantasy', 'Adventure'], + 'The Two Towers': ['Fantasy', 'Epic Fantasy', 'Adventure'], + 'The Last Wish': ['Fantasy', 'Sword and Sorcery', 'Adventure'], + 'Hyperion': ['Science Fiction', 'Space Opera', 'Adventure'], + 'The Martian': ['Science Fiction', 'Hard Science Fiction', 'Adventure'], + 'Foundation': ['Science Fiction', 'Space Opera'], +}; + +export let SEEDED_GENRES: Genre[] = []; + +export async function seed(knex: Knex): Promise { + await knex('book_genre').del(); + await knex('genre').del(); + + // Create all unique genres + const genres = await Promise.all( + GENRES.map((name) => createGenre(db, { name })) + ); + + SEEDED_GENRES = genres; + + // Create book-genre associations + const bookGenrePromises: Promise[] = []; + + SEEDED_BOOKS.forEach((book) => { + // Find matching genres for this book + const bookGenres = Object.entries(BOOK_GENRE_MAPPING).find(([titlePattern]) => + book.title.includes(titlePattern) + )?.[1] || []; + + // Associate book with its genres + bookGenres.forEach((genreName) => { + const genre = SEEDED_GENRES.find((g) => g.name === genreName); + if (genre) { + bookGenrePromises.push( + db('book_genre').insert({ + book_md5: book.md5, + genre_id: genre.id, + }) + ); + } + }); + }); + + await Promise.all(bookGenrePromises); + + console.log(`✓ Seeded ${SEEDED_GENRES.length} genres with book associations`); +} diff --git a/apps/server/src/db/seeds/07_users_progress.ts b/apps/server/src/db/seeds/07_users_progress.ts new file mode 100644 index 00000000..5fc33791 --- /dev/null +++ b/apps/server/src/db/seeds/07_users_progress.ts @@ -0,0 +1,72 @@ +import { Progress, User } from '@koinsight/common/types'; +import { Knex } from 'knex'; +import { db } from '../../knex'; +import { createUser } from '../factories/user-factory'; +import { createProgress } from '../factories/progress-factory'; +import { SEEDED_BOOKS } from './02_books'; +import { SEEDED_DEVICES } from './01_devices'; + +/** + * Create test users with the KoSync protocol + * Password for all test users is: password123 + */ +const TEST_USERS = [ + { username: 'reader1', password: 'password123' }, + { username: 'reader2', password: 'password123' }, + { username: 'bookworm', password: 'password123' }, +]; + +export let SEEDED_USERS: User[] = []; +export let SEEDED_PROGRESS: Progress[] = []; + +export async function seed(knex: Knex): Promise { + await knex('progress').del(); + await knex('user').del(); + + // Create test users + const users = await Promise.all( + TEST_USERS.map((userData) => createUser(db, userData)) + ); + + SEEDED_USERS = users; + + // Create progress records for each user + const progressPromises: Promise[] = []; + + SEEDED_USERS.forEach((user) => { + // Each user has progress for 3-7 random books + const numBooks = Math.floor(Math.random() * 5) + 3; + const userBooks = [...SEEDED_BOOKS] + .sort(() => Math.random() - 0.5) + .slice(0, numBooks); + + userBooks.forEach((book) => { + // Use a random device for each book + const device = SEEDED_DEVICES[Math.floor(Math.random() * SEEDED_DEVICES.length)]; + + // Create a realistic document name (similar to what KoReader uses) + const documentName = `${book.title.replace(/\s+/g, '_')}.epub`; + + // Random reading progress (0-100%) + const percentage = Math.random() * 100; + + progressPromises.push( + createProgress(db, user, device, documentName, { + percentage, + progress: JSON.stringify({ + page: Math.floor((book.reference_pages ?? 300) * (percentage / 100)), + position: percentage / 100, + }), + }) + ); + }); + }); + + SEEDED_PROGRESS = await Promise.all(progressPromises); + + console.log(`✓ Seeded ${SEEDED_USERS.length} users with ${SEEDED_PROGRESS.length} progress records`); + console.log(' Test user credentials:'); + TEST_USERS.forEach((user) => { + console.log(` - Username: ${user.username}, Password: ${user.password}`); + }); +} diff --git a/package.json b/package.json index e9a209ad..f546af54 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ "scripts": { "build": "turbo run build", "test:coverage": "turbo run test:coverage", - "dev": "turbo run dev --parallel" + "dev": "turbo run dev --parallel", + "seed": "npm --workspace=server run seed" }, "devDependencies": { "prettier": "3.6.2",