diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index dde42a0..3dc635e 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -151,10 +151,10 @@ Enable these settings in your repository: Add these badges to your README: ```markdown -[![CI](https://github.com/idotta/jsonb-store/actions/workflows/ci.yml/badge.svg)](https://github.com/idotta/jsonb-store/actions/workflows/ci.yml) -[![Code Quality](https://github.com/idotta/jsonb-store/actions/workflows/code-quality.yml/badge.svg)](https://github.com/idotta/jsonb-store/actions/workflows/code-quality.yml) +[![CI](https://github.com/idotta/lite-doc-store/actions/workflows/ci.yml/badge.svg)](https://github.com/idotta/lite-doc-store/actions/workflows/ci.yml) +[![Code Quality](https://github.com/idotta/lite-doc-store/actions/workflows/code-quality.yml/badge.svg)](https://github.com/idotta/lite-doc-store/actions/workflows/code-quality.yml) [![NuGet](https://img.shields.io/nuget/v/LiteDocumentStore.svg)](https://www.nuget.org/packages/LiteDocumentStore/) -[![codecov](https://codecov.io/gh/idotta/jsonb-store/branch/main/graph/badge.svg)](https://codecov.io/gh/idotta/jsonb-store) +[![codecov](https://codecov.io/gh/idotta/lite-doc-store/branch/main/graph/badge.svg)](https://codecov.io/gh/idotta/lite-doc-store) ``` ## Troubleshooting diff --git a/.github/instructions/code-style.instructions.md b/.github/instructions/code-style.instructions.md index a0cbc24..7b81ec1 100644 --- a/.github/instructions/code-style.instructions.md +++ b/.github/instructions/code-style.instructions.md @@ -86,9 +86,10 @@ SELECT * FROM [Customer] WHERE json_extract(data, '$.email') = @Email ### Parameterization ```csharp -// ALWAYS use parameters, never string concatenation -var sql = "SELECT * FROM [Customer] WHERE id = @Id"; -await _connection.QueryAsync(sql, new { Id = id }); +// ALWAYS use parameters, never string concatenation. +// Use the raw ADO.NET helpers in Core/SqliteCommandExtensions.cs (bind by name): +var sql = "SELECT json(data) FROM [Customer] WHERE id = @Id"; +var json = await _connection.QueryFirstStringAsync(sql, ("Id", id)); // NEVER do this var sql = $"SELECT * FROM [Customer] WHERE id = '{id}'"; // SQL INJECTION! diff --git a/.github/instructions/general.instructions.md b/.github/instructions/general.instructions.md index 7b7d64f..b99d51c 100644 --- a/.github/instructions/general.instructions.md +++ b/.github/instructions/general.instructions.md @@ -12,9 +12,8 @@ The goal is NOT an opaque document database. Users should seamlessly mix documen - **.NET 10** - Target framework - **SQLite 3.45+** - Required for JSONB support -- **Microsoft.Data.Sqlite** - SQLite provider -- **Dapper** - Micro-ORM for minimal mapping overhead -- **System.Text.Json** - Fixed, optimized JSON serializer +- **Microsoft.Data.Sqlite** - SQLite provider (raw ADO.NET; no ORM) +- **System.Text.Json** - AOT-safe serializer via `JsonTypeInfo` ## Key Architectural Principles @@ -30,9 +29,9 @@ The goal is NOT an opaque document database. Users should seamlessly mix documen ``` src/ ├── LiteDocumentStore/ # Main library -│ ├── DocumentStore.cs # Core repository implementation -│ ├── SqliteJsonbTypeHandler.cs # Dapper type handler for JSONB -│ └── JsonHelper.cs # Optimized JSON serialization +│ ├── Core/DocumentStore.cs # Core store implementation +│ ├── Core/SqliteCommandExtensions.cs # Raw ADO.NET helpers (replaced Dapper) +│ └── Serialization/JsonHelper.cs # AOT-safe JSON serialization └── tests/ ├── LiteDocumentStore.UnitTests/ # Unit tests (mocked) └── LiteDocumentStore.IntegrationTests/ # Integration tests (real SQLite) @@ -41,7 +40,8 @@ src/ ## When Modifying This Project -- Check `IMPLEMENTATION_CHECKLIST.md` for the roadmap +- See `CLAUDE.md` at the repo root for the authoritative architecture guide +- Keep the library AOT-clean: no reflection-based serialization, no `Expression.Compile`, no `dynamic` - All public APIs need XML documentation - New features need both unit and integration tests - Consider performance implications of every change diff --git a/.github/instructions/patterns.instructions.md b/.github/instructions/patterns.instructions.md deleted file mode 100644 index 4bf922b..0000000 --- a/.github/instructions/patterns.instructions.md +++ /dev/null @@ -1,227 +0,0 @@ -# Common Patterns Instructions - -## Repository Pattern - -The core `Repository` class provides document-style operations while exposing the connection for relational queries. - -### Basic CRUD - -```csharp -using var repo = new Repository("app.db"); - -// Create table (once per type) -await repo.CreateTableAsync(); - -// Create/Update -await repo.UpsertAsync("cust-123", new Customer { Name = "John" }); - -// Read -var customer = await repo.GetAsync("cust-123"); - -// Read all -var allCustomers = await repo.GetAllAsync(); - -// Delete -var deleted = await repo.DeleteAsync("cust-123"); -``` - -### Hybrid Relational Access - -```csharp -// Direct SQL via Dapper (always available) -var results = await repo.Connection.QueryAsync( - @"SELECT json_extract(data, '$.name') as Name, - json_extract(data, '$.email') as Email - FROM Customer - WHERE json_extract(data, '$.active') = 1"); - -// Join document tables -var orderDetails = await repo.Connection.QueryAsync( - @"SELECT o.id, json_extract(o.data, '$.total') as Total, - json_extract(c.data, '$.name') as CustomerName - FROM [Order] o - JOIN Customer c ON json_extract(o.data, '$.customerId') = c.id"); -``` - -## Transaction Pattern - -Batch operations for performance: - -```csharp -await repo.ExecuteInTransactionAsync(async () => -{ - foreach (var customer in customers) - { - await repo.UpsertAsync(customer.Id, customer); - } -}); -``` - -With transaction access: - -```csharp -await repo.ExecuteInTransactionAsync(async (transaction) => -{ - // Multiple operations in same transaction - await repo.UpsertAsync("cust-1", customer1); - await repo.UpsertAsync("cust-2", customer2); - - // Can also use Dapper directly with transaction - await repo.Connection.ExecuteAsync( - "UPDATE Customer SET data = jsonb(@Data) WHERE id = @Id", - new { Id = "cust-1", Data = json }, - transaction); -}); -``` - -## JSON Path Indexing Pattern - -Create indexes on frequently queried JSON fields: - -```csharp -// Create an index on the email field -await repo.Connection.ExecuteAsync( - @"CREATE INDEX IF NOT EXISTS idx_customer_email - ON Customer(json_extract(data, '$.email'))"); - -// Now this query uses the index -var customer = await repo.Connection.QueryFirstOrDefaultAsync( - @"SELECT json(data) as data FROM Customer - WHERE json_extract(data, '$.email') = @Email", - new { Email = "john@example.com" }); -``` - -## Virtual Column Pattern - -For frequently queried fields, use generated columns: - -```csharp -// Add a virtual column (one-time migration) -await repo.Connection.ExecuteAsync( - @"ALTER TABLE Customer - ADD COLUMN email TEXT - GENERATED ALWAYS AS (json_extract(data, '$.email')) VIRTUAL"); - -// Create index on virtual column -await repo.Connection.ExecuteAsync( - "CREATE INDEX IF NOT EXISTS idx_customer_email ON Customer(email)"); - -// Query uses standard column syntax -var customer = await repo.Connection.QueryFirstOrDefaultAsync( - "SELECT json(data) FROM Customer WHERE email = @Email", - new { Email = "john@example.com" }); -``` - -## Bulk Insert Pattern - -For large datasets, use transactions with batching: - -```csharp -public async Task BulkUpsertAsync(IEnumerable<(string Id, T Data)> items, int batchSize = 1000) -{ - var batches = items.Chunk(batchSize); - - foreach (var batch in batches) - { - await repo.ExecuteInTransactionAsync(async () => - { - foreach (var (id, data) in batch) - { - await repo.UpsertAsync(id, data); - } - }); - } -} -``` - -## Connection Reuse Pattern - -For high-throughput scenarios, reuse the repository: - -```csharp -// Singleton or scoped lifetime in DI -public class CustomerService -{ - private readonly Repository _repository; - - public CustomerService(Repository repository) - { - _repository = repository; - } - - public async Task GetCustomerAsync(string id) - { - return await _repository.GetAsync(id); - } -} -``` - -## Mixed Schema Pattern - -Combine document tables with traditional relational tables: - -```csharp -// Document table for flexible data -await repo.CreateTableAsync(); - -// Traditional relational table for structured data -await repo.Connection.ExecuteAsync(@" - CREATE TABLE IF NOT EXISTS audit_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - entity_type TEXT NOT NULL, - entity_id TEXT NOT NULL, - action TEXT NOT NULL, - timestamp INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), - user_id TEXT - )"); - -// Query across both -var auditWithCustomer = await repo.Connection.QueryAsync(@" - SELECT a.*, json_extract(c.data, '$.name') as CustomerName - FROM audit_log a - LEFT JOIN Customer c ON a.entity_id = c.id - WHERE a.entity_type = 'Customer'"); -``` - -## Error Handling Pattern - -```csharp -try -{ - await repo.UpsertAsync(id, data); -} -catch (SqliteException ex) when (ex.SqliteErrorCode == 19) // CONSTRAINT -{ - // Handle constraint violation - throw new DuplicateKeyException($"Record with ID {id} already exists", ex); -} -catch (SqliteException ex) when (ex.SqliteErrorCode == 11) // CORRUPT -{ - // Handle database corruption - _logger.LogCritical(ex, "Database corruption detected"); - throw; -} -``` - -## Optimistic Concurrency Pattern - -```csharp -// Table with version column -await repo.Connection.ExecuteAsync(@" - CREATE TABLE IF NOT EXISTS [Customer] ( - id TEXT PRIMARY KEY, - data BLOB NOT NULL, - version INTEGER NOT NULL DEFAULT 1 - )"); - -// Update with version check -var affected = await repo.Connection.ExecuteAsync(@" - UPDATE [Customer] - SET data = jsonb(@Data), - version = version + 1 - WHERE id = @Id AND version = @ExpectedVersion", - new { Id = id, Data = json, ExpectedVersion = version }); - -if (affected == 0) - throw new ConcurrencyException("Record was modified by another process"); -``` diff --git a/.github/instructions/testing.instructions.md b/.github/instructions/testing.instructions.md deleted file mode 100644 index 351411d..0000000 --- a/.github/instructions/testing.instructions.md +++ /dev/null @@ -1,172 +0,0 @@ -# Testing Instructions - -## Testing Framework - -- **xUnit** - Test framework -- **Moq** - Mocking library (if needed) - -## Test Project Structure - -``` -tests/ -├── LiteDocumentStore.UnitTests/ # Fast, isolated tests -│ └── RepositoryTests.cs -└── LiteDocumentStore.IntegrationTests/ # Real database tests - └── RepositoryIntegrationTests.cs -``` - -## Unit Tests - -Unit tests should be fast and isolated. Mock external dependencies. - -```csharp -public class RepositoryTests -{ - [Fact] - public void GetTableName_ReturnsTypeName() - { - // Arrange & Act - var result = Repository.GetTableName(); - - // Assert - result.Should().Be("Customer"); - } - - [Theory] - [InlineData(null)] - [InlineData("")] - public async Task GetAsync_WithInvalidId_ThrowsArgumentException(string id) - { - // Arrange - using var repo = new Repository(":memory:"); - - // Act & Assert - await Assert.ThrowsAsync( - () => repo.GetAsync(id)); - } -} -``` - -## Integration Tests - -Integration tests use real SQLite databases. Prefer in-memory for speed. - -```csharp -public class RepositoryIntegrationTests : IAsyncLifetime -{ - private Repository _repository = null!; - - public async Task InitializeAsync() - { - // Use in-memory database for speed - _repository = new Repository(":memory:"); - await _repository.CreateTableAsync(); - } - - public async Task DisposeAsync() - { - await _repository.DisposeAsync(); - } - - [Fact] - public async Task UpsertAndGet_RoundTrip_PreservesData() - { - // Arrange - var customer = new Customer { Name = "John", Email = "john@test.com" }; - - // Act - await _repository.UpsertAsync("cust-1", customer); - var retrieved = await _repository.GetAsync("cust-1"); - - // Assert - retrieved.Should().NotBeNull(); - retrieved!.Name.Should().Be("John"); - retrieved.Email.Should().Be("john@test.com"); - } -} -``` - -## Test Naming Convention - -Use descriptive names following the pattern: -`MethodName_Scenario_ExpectedBehavior` - -```csharp -[Fact] -public async Task GetAsync_WhenIdNotFound_ReturnsNull() - -[Fact] -public async Task UpsertAsync_WithExistingId_UpdatesRecord() - -[Fact] -public async Task DeleteAsync_WhenRecordExists_ReturnsTrue() -``` - -## Test Categories - -Use traits to categorize tests: - -```csharp -[Trait("Category", "Unit")] -public class RepositoryTests { } - -[Trait("Category", "Integration")] -public class RepositoryIntegrationTests { } - -[Trait("Category", "Performance")] -public class RepositoryBenchmarks { } -``` - -## What to Test - -### Unit Tests -- Input validation -- Edge cases (null, empty, special characters) -- Error conditions -- Pure logic methods - -### Integration Tests -- CRUD operations round-trip -- Transaction behavior -- Concurrent access patterns -- WAL mode behavior -- JSONB serialization/deserialization -- Index creation and usage -- Large dataset handling - -## Test Data - -Create test fixtures for reusable test data: - -```csharp -public static class TestData -{ - public static Customer CreateCustomer(string id = "cust-1") => new() - { - Id = id, - Name = "Test Customer", - Email = "test@example.com", - CreatedAt = DateTime.UtcNow - }; - - public static IEnumerable CreateCustomers(int count) => - Enumerable.Range(1, count) - .Select(i => CreateCustomer($"cust-{i}")); -} -``` - -## Running Tests - -```bash -# Run all tests -dotnet test - -# Run only unit tests -dotnet test --filter "Category=Unit" - -# Run only integration tests -dotnet test --filter "Category=Integration" - -# Run with coverage -dotnet test --collect:"XPlat Code Coverage" -``` diff --git a/AOT_COMPATIBILITY_PLAN.md b/AOT_COMPATIBILITY_PLAN.md deleted file mode 100644 index 1bae3bf..0000000 --- a/AOT_COMPATIBILITY_PLAN.md +++ /dev/null @@ -1,700 +0,0 @@ -# AOT Compatibility Plan for LiteDocumentStore - -**Date:** January 12, 2026 -**Status:** Analysis & Planning Phase - -## Executive Summary - -This document outlines the strategy for making LiteDocumentStore compatible with .NET Native AOT compilation. The primary goal is to eliminate reflection-based patterns that prevent AOT compilation and improve benchmark performance through compile-time code generation. - -**Expected Benefits:** -- 30-50% faster startup time -- 20-40% lower memory usage -- 10-20% faster query execution -- Smaller deployment size for single-file publish -- iOS/Android native compilation support - ---- - -## Current AOT Blockers - -### 1. JSON Serialization (CRITICAL) - -**Location:** `src/LiteDocumentStore/Serialization/JsonHelper.cs` - -**Problem:** -```csharp -// Uses reflection-based serialization -public static byte[] SerializeToUtf8Bytes(T value) -{ - return JsonSerializer.SerializeToUtf8Bytes(value, Options); -} -``` - -**Impact:** HIGH - Used in every read/write operation throughout the codebase -- `DocumentStore.UpsertAsync()` -- `DocumentStore.GetAsync()` -- `DocumentStore.QueryAsync()` -- All benchmark operations - -**Root Cause:** `System.Text.Json` without source generation relies on reflection to discover type members at runtime. - ---- - -### 2. Expression Tree Reflection (CRITICAL) - -**Location:** `src/LiteDocumentStore/Core/ExpressionToJsonPath.cs` - -**Problem:** -```csharp -private static object EvaluateMemberExpression(MemberExpression memberExpr) -{ - // Uses reflection to walk member chains - var member = memberChain[i]; - value = member switch - { - FieldInfo field => field.GetValue(value), // ❌ Reflection - PropertyInfo property => property.GetValue(value), // ❌ Reflection - _ => throw new NotSupportedException() - }; -} - -private static object EvaluateMethodCall(MethodCallExpression methodCall) -{ - // Uses reflection to invoke methods - var result = methodCall.Method.Invoke(instance, args); // ❌ Reflection -} -``` - -**Impact:** HIGH - Core querying and indexing functionality -- `QueryAsync(Expression> predicate)` -- `CreateIndexAsync(Expression> jsonPath)` -- `CreateCompositeIndexAsync()` -- `AddVirtualColumnAsync()` -- `SelectAsync()` - -**Additional Issues in ExpressionToJsonPath:** -- Line 308-325: `EvaluateMemberExpression` uses `FieldInfo.GetValue()` and `PropertyInfo.GetValue()` -- Line 336-352: `EvaluateMethodCall` uses `MethodInfo.Invoke()` -- Heavy use of `System.Reflection` namespace - ---- - -### 3. Dapper Micro-ORM (HIGH) - -**Location:** Used throughout `DocumentStore.cs` - -**Problem:** -```csharp -// Dapper uses reflection for object mapping -var json = await _connection.QueryFirstOrDefaultAsync(sql, new { Id = id }); -var results = await _connection.QueryAsync(sql, parameters); -``` - -**Impact:** HIGH - Fundamental data access layer - -**Affected Methods:** -- All database query operations -- Parameter binding with anonymous types -- Result set mapping - ---- - -### 4. Type Handler Registration (MEDIUM) - -**Location:** `src/LiteDocumentStore/TypeHandlers/DateTimeOffsetHandler.cs` - -**Problem:** -```csharp -public sealed class DateTimeOffsetHandler : SqlMapper.TypeHandler -{ - public override DateTimeOffset Parse(object value) - { - // Type checking at runtime - if (value is string strValue) { ... } - throw new SerializationException($"Unsupported type: {value.GetType().Name}"); // ❌ - } -} - -// Registration in MigrationRunner.cs: -SqlMapper.AddTypeHandler(new DateTimeOffsetHandler()); // ❌ Dapper's TypeHandler system -``` - -**Impact:** MEDIUM - Specific to DateTimeOffset handling, but Dapper's type handler system uses reflection - ---- - -### 5. typeof() Usage in Error Messages (LOW) - -**Location:** Multiple files - -**Problem:** -```csharp -// JsonHelper.cs -throw new SerializationException( - $"Failed to serialize object of type {typeof(T).Name}.", // ⚠️ Requires type metadata - typeof(T), - ex); -``` - -**Impact:** LOW - Primarily diagnostic, but requires type metadata to be preserved - ---- - -## Implementation Strategy - -### Phase 1: JSON Serialization Source Generation - -#### 1.1 Create User-Provided JSON Context Pattern - -Since the library needs to serialize arbitrary user types, we require users to provide their own `JsonSerializerContext`: - -**New API Design:** -```csharp -// User defines their context -[JsonSerializable(typeof(Person))] -[JsonSerializable(typeof(Order))] -[JsonSerializable(typeof(Product))] -public partial class MyAppJsonContext : JsonSerializerContext -{ -} - -// User configures DocumentStore -var options = new DocumentStoreOptions -{ - JsonContext = MyAppJsonContext.Default -}; - -var store = await DocumentStoreFactory.CreateAsync( - connectionString, - options); -``` - -#### 1.2 Create AOT-Compatible JsonHelper - -**New File:** `src/LiteDocumentStore/Serialization/AotJsonHelper.cs` - -```csharp -internal static class AotJsonHelper -{ - public static byte[] SerializeToUtf8Bytes(T value, JsonSerializerContext context) - { - var typeInfo = (JsonTypeInfo)context.GetTypeInfo(typeof(T)); - return JsonSerializer.SerializeToUtf8Bytes(value, typeInfo); - } - - public static T? Deserialize(ReadOnlySpan utf8Json, JsonSerializerContext context) - { - var typeInfo = (JsonTypeInfo)context.GetTypeInfo(typeof(T)); - return JsonSerializer.Deserialize(utf8Json, typeInfo); - } -} -``` - -#### 1.3 Update DocumentStoreOptions - -```csharp -public class DocumentStoreOptions -{ - // Existing options... - - /// - /// Optional JSON serialization context for AOT compatibility. - /// When provided, uses source-generated serialization instead of reflection. - /// Required for Native AOT scenarios. - /// - public JsonSerializerContext? JsonContext { get; set; } -} -``` - -**Migration Path:** -- Keep existing `JsonHelper` for backward compatibility -- Add optional `JsonContext` to options -- `DocumentStore` checks if `JsonContext` is provided and uses appropriate helper - ---- - -### Phase 2: Replace Expression Tree Reflection - -This is the most complex transformation. We have three approaches: - -#### Approach A: Source Generators for JSON Paths (RECOMMENDED) - -**Generator:** Analyzes types and generates compile-time JSON path constants - -**User Code:** -```csharp -// User writes the same code -await store.CreateIndexAsync(x => x.Email); -await store.CreateIndexAsync(x => x.Address.City); -``` - -**Generated Code:** -```csharp -// Auto-generated: Person.JsonPaths.g.cs -namespace LiteDocumentStore.Generated; - -[GeneratedCode("LiteDocumentStore.SourceGenerator", "1.0.0")] -internal static partial class JsonPaths -{ - public static partial class Person - { - public static string Email => "$.Email"; - public static string Name => "$.Name"; - public static string Age => "$.Age"; - public static string Address => "$.Address"; - public static string Address_City => "$.Address.City"; - public static string Address_State => "$.Address.State"; - } -} -``` - -**Implementation using C# Interceptors:** -```csharp -// Generator intercepts the method call -[InterceptsLocation("DocumentStore.cs", line: 420, character: 15)] -public static Task CreateIndexAsync_Person_Email( - this IDocumentStore store, - Expression> jsonPath, - string? indexName = null) -{ - // Direct path, no reflection - return store.CreateIndexAsync_Internal("$.Email", indexName); -} -``` - -**Pros:** -- ✅ Maintains current API surface -- ✅ Type-safe at compile time -- ✅ Zero runtime reflection -- ✅ No user API changes - -**Cons:** -- ⚠️ Requires C# 12+ for interceptors -- ⚠️ Complex generator implementation - ---- - -#### Approach B: String-Based API (SIMPLE) - -**New API:** -```csharp -// Replace expression-based methods with string-based equivalents -await store.CreateIndexAsync("$.Email"); -await store.CreateIndexAsync("$.Address.City"); -await store.AddVirtualColumnAsync("$.Email", "email_column"); -``` - -**Pros:** -- ✅ Simple implementation -- ✅ No reflection required -- ✅ Works with all .NET versions - -**Cons:** -- ❌ Loses type safety -- ❌ Breaking API change -- ❌ More error-prone - ---- - -#### Approach C: Builder Pattern with Source Generation - -**Generated Builder:** -```csharp -// Generated: PersonQueryBuilder.g.cs -public class PersonQueryBuilder -{ - public PersonQueryBuilder WhereEmail(string value) { ... } - public PersonQueryBuilder WhereAge(int value) { ... } - public PersonQueryBuilder WhereAgeGreaterThan(int value) { ... } -} - -// Usage: -var query = new PersonQueryBuilder() - .WhereAgeGreaterThan(30) - .WhereEmail("john@example.com") - .Build(); - -await store.QueryAsync(query); -``` - -**Pros:** -- ✅ Type-safe -- ✅ No reflection -- ✅ Discoverable API - -**Cons:** -- ⚠️ Different API surface -- ⚠️ Verbose for complex queries - ---- - -### Phase 3: Replace Dapper - -#### Option 1: Dapper.AOT (Microsoft Official) - -**Status:** Preview/Experimental - -```csharp -// Uses Dapper's source generator -[DapperAot] -public interface IPersonRepository -{ - [Query("SELECT json(data) FROM Person WHERE id = @Id")] - Task GetByIdAsync(string id); -} -``` - -**Investigation Needed:** -- Stability and performance -- Integration with existing code -- Binary size impact - ---- - -#### Option 2: Custom Source Generator for SQL Mapping - -**Generator creates typed mapping code:** - -```csharp -// Current Dapper code: -var results = await _connection.QueryAsync(sql, new { Value = value }); - -// Generated alternative: -var results = await SqlMapper_Person_QueryByEmail( - _connection, - sql, - new QueryParameters { Value = value }); - -// Generator produces: -[GeneratedCode] -private static async Task> SqlMapper_Person_QueryByEmail( - SqliteConnection connection, - string sql, - QueryParameters parameters) -{ - using var command = connection.CreateCommand(); - command.CommandText = sql; - command.Parameters.AddWithValue("@Value", parameters.Value); - - var results = new List(); - using var reader = await command.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - { - results.Add(reader.GetString(0)); - } - return results; -} -``` - -**Pros:** -- ✅ Full control over implementation -- ✅ Optimized for our specific use case -- ✅ No external dependencies - -**Cons:** -- ⚠️ Significant development effort -- ⚠️ Maintenance burden - ---- - -#### Option 3: Manual ADO.NET with Helper Extensions - -**Simplest approach - manually write data access code:** - -```csharp -// Replace Dapper calls with manual ADO.NET -private async Task GetAsync(string id) -{ - using var command = _connection.CreateCommand(); - command.CommandText = SqlGenerator.GenerateGetByIdSql(tableName); - command.Parameters.AddWithValue("@Id", id); - - var json = await ExecuteScalarAsync(command); - return json != null ? JsonHelper.Deserialize(json) : default; -} -``` - -**Pros:** -- ✅ Simple and maintainable -- ✅ No reflection -- ✅ Small code footprint - -**Cons:** -- ⚠️ More boilerplate code -- ⚠️ Loses Dapper's convenience - ---- - -## Recommended Architecture - -### Parallel Package Approach (RECOMMENDED) - -Create a separate `LiteDocumentStore.Aot` package: - -``` -LiteDocumentStore.Aot/ -├── Core/ -│ ├── AotDocumentStore.cs # AOT-compatible implementation -│ ├── AotDocumentStoreOptions.cs # Options requiring JsonContext -│ └── IAotDocumentStore.cs # AOT-specific interface -├── SourceGenerators/ -│ ├── JsonPathSourceGenerator.cs # Generates JSON path constants -│ ├── QueryInterceptorGenerator.cs # Intercepts expression methods -│ ├── SqlMappingGenerator.cs # Generates SQL mappers -│ └── SerializationContextGenerator.cs # Helper for user contexts -├── Serialization/ -│ └── AotJsonHelper.cs # Context-based serialization -└── Querying/ - ├── QueryBuilder.cs # Fallback builder API - └── QueryExpression.cs # Runtime query model -``` - -**Benefits:** -- ✅ Maintains backward compatibility in main package -- ✅ Clear separation of concerns -- ✅ Users opt-in to AOT constraints -- ✅ Different API contracts for AOT - -**Migration Story:** -```csharp -// Before (LiteDocumentStore): -var store = await DocumentStoreFactory.CreateAsync("app.db"); -await store.UpsertAsync("id", person); -await store.QueryAsync(p => p.Age > 30); - -// After (LiteDocumentStore.Aot): -var store = await AotDocumentStoreFactory.CreateAsync( - "app.db", - new AotDocumentStoreOptions - { - JsonContext = MyAppJsonContext.Default - }); -await store.UpsertAsync("id", person); -await store.QueryAsync(p => p.Age > 30); // Intercepted by generator -``` - ---- - -## Implementation Roadmap - -### Milestone 1: JSON Serialization (2-3 weeks) -- [ ] Design `DocumentStoreOptions.JsonContext` API -- [ ] Implement `AotJsonHelper` with context support -- [ ] Update `DocumentStore` to conditionally use AOT helper -- [ ] Create example app with custom `JsonSerializerContext` -- [ ] Write integration tests -- [ ] **Success Metric:** Can serialize/deserialize with source-generated context - -### Milestone 2: Expression Interceptors (4-6 weeks) -- [ ] Create source generator project -- [ ] Implement JSON path generator -- [ ] Implement method interceptors for `CreateIndexAsync`, `AddVirtualColumnAsync` -- [ ] Handle nested properties and edge cases -- [ ] Write generator tests -- [ ] **Success Metric:** Can create indexes without expression tree evaluation - -### Milestone 3: Query Translation (4-6 weeks) -- [ ] Design query interceptor strategy -- [ ] Implement `QueryAsync` interceptor -- [ ] Support equality, comparison, and logical operators -- [ ] Support string methods (Contains, StartsWith, EndsWith) -- [ ] Fallback for complex queries -- [ ] **Success Metric:** Can execute common queries without reflection - -### Milestone 4: Replace Dapper (3-4 weeks) -- [ ] Evaluate Dapper.AOT vs custom generator -- [ ] Implement SQL mapping generator (if custom) -- [ ] Migrate all Dapper calls -- [ ] Performance testing -- [ ] **Success Metric:** All database operations work without Dapper reflection - -### Milestone 5: Testing & Benchmarking (2-3 weeks) -- [ ] Create comprehensive test suite -- [ ] Port all existing tests to AOT version -- [ ] Create AOT-specific benchmarks -- [ ] Compare startup time, memory, throughput -- [ ] Test Native AOT publish scenarios -- [ ] **Success Metric:** AOT version passes all tests and shows performance improvements - -### Milestone 6: Documentation & Release (1-2 weeks) -- [ ] Update README with AOT guidance -- [ ] Create migration guide -- [ ] Document API differences -- [ ] Release preview NuGet package -- [ ] Gather community feedback - -**Total Estimated Time:** 16-24 weeks (4-6 months) - ---- - -## Expected Performance Improvements - -Based on AOT best practices and similar library migrations: - -| Metric | Current (Reflection) | AOT-Compatible | Improvement | -|--------|---------------------|----------------|-------------| -| **Cold Start Time** | ~200ms | ~80ms | **60% faster** | -| **Memory (Startup)** | ~15MB | ~9MB | **40% reduction** | -| **Query Execution** | 100µs | 85µs | **15% faster** | -| **Serialization** | 50µs | 35µs | **30% faster** | -| **Binary Size** | 8MB | 12MB | 50% larger | -| **Trim Compatibility** | Partial | Full | ✅ | - -**Benchmark Priorities:** -1. Startup time (most visible improvement) -2. Memory footprint (important for serverless/mobile) -3. Query throughput (less dramatic but measurable) - ---- - -## Trade-offs & Considerations - -### API Ergonomics -- **Current:** Excellent - natural LINQ expressions -- **AOT:** Good - interceptors maintain API, but some limitations -- **Mitigation:** Provide builder pattern as fallback - -### Development Complexity -- **Current:** Moderate - straightforward reflection patterns -- **AOT:** High - source generators, interceptors, advanced concepts -- **Mitigation:** Comprehensive documentation and examples - -### Binary Size -- **Current:** Small - minimal code generation -- **AOT:** Larger - all code pre-compiled -- **Mitigation:** Acceptable trade-off for AOT scenarios - -### Backward Compatibility -- **Approach:** Separate package maintains full compatibility -- **Risk:** Low - existing users unaffected -- **Benefit:** Can evolve AOT API independently - ---- - -## Testing Strategy - -### Unit Tests -- [ ] All existing unit tests must pass with AOT implementation -- [ ] Add AOT-specific tests for source generators -- [ ] Test interceptor edge cases - -### Integration Tests -- [ ] Port all integration tests to AOT version -- [ ] Test actual Native AOT publish -- [ ] Test on multiple platforms (Windows, Linux, macOS) - -### Benchmark Tests -- [ ] Create side-by-side comparison benchmarks -- [ ] Measure startup time, memory, throughput -- [ ] Compare binary sizes -- [ ] Test in realistic scenarios (web API, CLI tool) - -### Compatibility Tests -- [ ] Test with various .NET versions (8.0+) -- [ ] Test Native AOT publish configurations -- [ ] Test trimming behavior - ---- - -## Open Questions - -1. **Dapper Replacement:** Should we use Dapper.AOT (if stable by then) or build custom? - - **Action:** Prototype both and compare - -2. **Interceptor Stability:** C# interceptors are experimental - risk assessment needed - - **Action:** Have builder pattern fallback ready - -3. **User Context Registration:** How do users register multiple document types? - - **Action:** Design clear registration pattern - -4. **Complex Query Support:** What's the fallback for queries that can't be intercepted? - - **Action:** Hybrid mode or clear error messages - -5. **Breaking Changes:** Can we maintain 100% API compatibility with interceptors? - - **Action:** Document any limitations clearly - ---- - -## Success Criteria - -### Must Have -- ✅ Full Native AOT compilation support -- ✅ No runtime reflection in hot paths -- ✅ All core features working (CRUD, queries, indexes) -- ✅ Measurable performance improvements (startup, memory) -- ✅ Comprehensive documentation - -### Should Have -- ✅ Type-safe API via interceptors (not just strings) -- ✅ 80%+ of existing API surface preserved -- ✅ Clear migration path from main package -- ✅ Example applications demonstrating usage - -### Nice to Have -- ⭐ iOS/Android compatibility verified -- ⭐ Blazor WASM support -- ⭐ Trimming analyzer support -- ⭐ Community adoption and feedback - ---- - -## Next Steps - -### Immediate Actions (This Week) -1. ✅ Complete this analysis document -2. Create `LiteDocumentStore.Aot` project structure -3. Implement basic `AotJsonHelper` with `JsonSerializerContext` support -4. Create sample app with custom serialization context -5. Prototype simple source generator for JSON paths - -### Short Term (This Month) -1. Validate source generator approach works -2. Benchmark JSON serialization (reflection vs source-generated) -3. Design interceptor strategy for expression methods -4. Begin Dapper replacement research - -### Medium Term (Next Quarter) -1. Implement core source generators -2. Port key functionality to AOT package -3. Create benchmark suite -4. Alpha testing with early adopters - ---- - -## References & Resources - -### Source Generation -- [System.Text.Json source generation](https://learn.microsoft.com/en-us/dotnet/standard/serialization/system-text-json/source-generation) -- [C# Source Generators](https://learn.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/source-generators-overview) -- [Interceptors in C# 12](https://learn.microsoft.com/en-us/dotnet/csharp/whats-new/csharp-12#interceptors) - -### Native AOT -- [Native AOT deployment](https://learn.microsoft.com/en-us/dotnet/core/deploying/native-aot/) -- [Prepare .NET libraries for trimming](https://learn.microsoft.com/en-us/dotnet/core/deploying/trimming/prepare-libraries-for-trimming) -- [AOT-compatible libraries](https://learn.microsoft.com/en-us/dotnet/core/deploying/native-aot/libraries) - -### Similar Migrations -- [Dapper.AOT](https://github.com/DapperLib/DapperAOT) -- [Entity Framework Core compiled models](https://learn.microsoft.com/en-us/ef/core/performance/advanced-performance-topics#compiled-models) -- [Refit source generator](https://github.com/reactiveui/refit) - ---- - -## Conclusion - -Making LiteDocumentStore AOT-compatible is a significant undertaking that requires: -- Source generators for JSON path extraction -- Method interceptors for expression-based APIs -- Replacement of Dapper's reflection-based mapping -- User-provided `JsonSerializerContext` for serialization - -The **recommended approach** is a parallel `LiteDocumentStore.Aot` package that maintains backward compatibility while providing full AOT support with measurable performance improvements. - -The investment will enable: -- Native mobile app compilation (iOS/Android) -- Faster serverless cold starts -- Lower memory footprint -- Future-proof architecture for .NET evolution - -**Estimated ROI:** High - AOT is the future of .NET deployment, and early adoption positions the library as a leader in the document store space. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..0caa279 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,106 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +LiteDocumentStore is a .NET library (published to NuGet as `LiteDocumentStore`) that turns a single SQLite `.db` file into a hybrid document + relational store. C# objects are serialized to JSON and stored in SQLite's **JSONB** binary format (requires SQLite 3.45+); the same tables stay fully accessible to raw SQL, joins, and indexes. The design goal is explicitly *not* an opaque document DB — users mix document-style CRUD and relational queries freely via the exposed `Connection`. + +Solution is `src/LiteDocumentStore.slnx`. Target is `net10.0`, `LangVersion=latest` (C# 14), nullable + implicit usings on. Data access is raw ADO.NET over `Microsoft.Data.Sqlite` (no Dapper — parameters bound explicitly, results read by ordinal); serialization is `System.Text.Json`. The library is Native-AOT / trim compatible (`true`). + +## Build, run, test + +Solution and all commands live under `src/` (CI sets `working-directory: ./src`). Run from there. + +```powershell +cd src + +# Build (Release is what CI uses) +dotnet build --configuration Release + +# Test — unit (fast, isolated) and integration (real in-memory SQLite) are separate projects +dotnet test tests/LiteDocumentStore.UnitTests/LiteDocumentStore.UnitTests.csproj +dotnet test tests/LiteDocumentStore.IntegrationTests/LiteDocumentStore.IntegrationTests.csproj + +# Single test / filter (tests are tagged with [Trait("Category", ...)] and named Method_Scenario_Expected) +dotnet test --filter "Category=Unit" +dotnet test --filter "FullyQualifiedName~UpsertAndGet_RoundTrip" + +# Benchmarks (BenchmarkDotNet) +dotnet run -c Release --project tests/LiteDocumentStore.Benchmarks +``` + +## Project layout + +``` +src/ + LiteDocumentStore.slnx Solution + LiteDocumentStore/ The library + Core/ DocumentStore, IDocumentStore, SqlGenerator, SqliteCommandExtensions + (raw ADO helpers), DocumentStoreOptions(+Builder) + Conventions/ ITableNamingConvention — maps type -> table name + Factories/ IDocumentStoreFactory, IConnectionFactory (+ Default impls) + Extensions/ ServiceCollectionExtensions (AddLiteDocumentStore, keyed variant) + Migrations/ MigrationRunner, IMigration/Migration, SchemaIntrospector + Serialization/ JsonHelper (STJ, via JsonTypeInfo) + Exceptions/ LiteDocumentStoreException + Concurrency/Serialization/TableNotFound + tests/ + LiteDocumentStore.UnitTests/ xUnit, mocked/isolated + LiteDocumentStore.IntegrationTests/ xUnit, real SQLite (mostly :memory:) + LiteDocumentStore.Benchmarks/ BenchmarkDotNet +``` + +`DocumentStore` and most internals are `internal sealed`; the test/benchmark projects see them via `InternalsVisibleTo` in the csproj. Consumers only touch the public surface: `IDocumentStore`, `DocumentStoreOptions`, the factories, and the DI extension. + +> Note: the `tests/` projects were updated for the Dapper removal (tests that only covered the dropped `QueryAsync(predicate)` / `SelectAsync` APIs were removed; the rest use the raw-ADO helpers in `Core/SqliteCommandExtensions.cs`). The `benchmarks/` project intentionally keeps a `Dapper` package reference as a *comparison baseline* only — it is not a library dependency. All three projects compile and `dotnet test` passes. + +## Documentation is stale — trust the code + +`README.md` and `.github/instructions/*.md` (Copilot rules) describe an older `Repository` class with a `new Repository("app.db")` constructor and a `SqliteJsonbTypeHandler`. **That API no longer exists.** The real entry point is `IDocumentStore`, obtained through DI (`services.AddLiteDocumentStore(...)`) or `IDocumentStoreFactory.Create/CreateAsync(DocumentStoreOptions)`. When those docs conflict with the source, the source wins. The *conceptual* guidance in those files (JSONB read/write pattern, WAL config, hybrid philosophy, SQLite error codes) is still accurate. + +## Architecture + +**All SQL is centralized in `SqlGenerator`** (static, one method per statement shape). Nothing else hand-writes SQL against document tables. Table schema is uniform: `id TEXT PRIMARY KEY, data BLOB NOT NULL`. The JSONB contract, enforced there, is load-bearing: +- **Write:** `jsonb(@Data)` — `@Data` is UTF-8 JSON *bytes* from `JsonHelper.SerializeToUtf8Bytes`, not a string. +- **Read:** `SELECT json(data)` — converts JSONB binary back to JSON text for deserialization. JSONB is binary; a raw `SELECT data` is not deserializable. +- Table names are always bracketed (`[{tableName}]`) and derived from the type name via `ITableNamingConvention` — never from user input, so there's no injection surface there. All *values* are parameterized. + +**Connection model.** A `DocumentStore` wraps one `SqliteConnection` and an `ownsConnection` flag. Via the factory it owns and manages the connection (opened + PRAGMAs applied by `DefaultConnectionFactory` from `DocumentStoreOptions`); when disposed it runs `PRAGMA wal_checkpoint(TRUNCATE)` before closing. The DI registration defaults to **Singleton** — one long-lived connection. Every method guards with `ObjectDisposedException.ThrowIf` + `EnsureConnectionOpen`. + +**Querying.** Documents are queried by JSON path + value via `QueryAsync(jsonPath, value)`, which builds `WHERE json_extract(data, '$.Path') = @Value`. The LINQ-predicate `QueryAsync(Expression>)` and the `SelectAsync` projections were **removed** (they required runtime reflection / IL generation that AOT can't support). `CreateIndexAsync`, `CreateCompositeIndexAsync`, and `AddVirtualColumnAsync` still accept LINQ expressions, but only walk **member names** (`DocumentStore.ExtractJsonPath`) to build `$.Path` — no compilation or closure evaluation, so they stay AOT-safe. Property names map **as-is (PascalCase)** to match default STJ serialization. For richer filtering (ranges, virtual-column index seeks, joins), drop to raw SQL via the `Connection` escape hatch. + +**Transactions.** `ExecuteInTransactionAsync` wraps `BeginTransaction` with commit/rollback. Commands are created with `SqliteConnection.CreateCommand`, which auto-enlists the connection's active transaction, so document operations invoked inside the action participate automatically. One transaction per connection (no nesting) — batch writes go through `UpsertManyAsync`/`DeleteManyAsync`, which build a single multi-row statement with explicit `@Id{i}`/`@Data{i}` parameters. + +**Migrations.** `MigrationRunner` tracks applied versions in a `__store_migrations` table; `IMigration` implementations provide `UpAsync`/`DownAsync`. Each apply/rollback is transactional. + +## AOT compatibility + +The library is Native-AOT / trim compatible as a **single package** (`true` turns on the trim + AOT analyzers). How each former blocker was handled: + +1. **Serialization** — `JsonHelper` goes through the AOT-safe `JsonTypeInfo` overloads, resolving type metadata from `DocumentStoreOptions.SerializerOptions`. AOT consumers pass options backed by a source-generated `JsonSerializerContext` (`new JsonSerializerOptions { TypeInfoResolver = MyContext.Default }`). When none is supplied, `JsonHelper.CreateDefaultReflectionOptions()` provides a reflection fallback — the single quarantined, `[UnconditionalSuppressMessage]`-annotated spot, used only in non-AOT scenarios. +2. **Dapper** — removed entirely, replaced by `Core/SqliteCommandExtensions.cs` (explicit parameter binding + ordinal reads). +3. **LINQ-predicate query + `SelectAsync` projections** — removed (needed reflection/`Expression.Compile`). `ExpressionToJsonPath` was deleted; the surviving expression APIs only read member names. +4. **`SchemaIntrospector`** — the `dynamic` PRAGMA read was rewritten to ordinal `DbDataReader` access. + +When adding features, keep them AOT-clean: no reflection-based serialization (route through `JsonHelper` + `JsonTypeInfo`), no `Expression.Compile`, no `dynamic`. A `dotnet build` must stay free of `IL2xxx`/`IL3xxx` warnings. + +## Conventions + +- File-scoped namespaces; `sealed` on non-inheritable classes; `readonly` fields; `_camelCase` private fields. Expression-bodied members for one-liners. +- Library code uses `.ConfigureAwait(false)` on awaits and `Async` suffix on async methods. +- Validate arguments up front and fail fast (`ArgumentException`/`ArgumentNullException.ThrowIfNull`). Rethrow inside `catch` on transaction rollback rather than wrapping. +- All public API needs XML doc comments (`GenerateDocumentationFile` is on; missing docs surface as warnings). +- Package versions are inline `` in the csproj — there is no Central Package Management here. +- New features need both a unit and an integration test. +- Don't add AI-attribution trailers (`Co-Authored-By: Claude`, "Generated with Claude Code") to commits or PRs. +- Never auto-commit — stage and commit only when the user explicitly asks. + +## Where to look first + +- `src/LiteDocumentStore/Core/DocumentStore.cs` — the whole implementation; every public operation lives here. +- `src/LiteDocumentStore/Core/SqlGenerator.cs` — the JSONB SQL contract; change SQL here, nowhere else. +- `src/LiteDocumentStore/Core/SqliteCommandExtensions.cs` — the raw ADO.NET helpers that replaced Dapper. +- `src/LiteDocumentStore/Serialization/JsonHelper.cs` — the AOT-safe serialization funnel (`JsonTypeInfo` + reflection fallback). +- `src/LiteDocumentStore/Extensions/ServiceCollectionExtensions.cs` — how consumers wire it up (DI + lifetimes). +- `examples/AotVerification.cs` — end-to-end AOT smoke test with a source-generated context. + diff --git a/IMPLEMENTATION_CHECKLIST.md b/IMPLEMENTATION_CHECKLIST.md deleted file mode 100644 index 372718b..0000000 --- a/IMPLEMENTATION_CHECKLIST.md +++ /dev/null @@ -1,328 +0,0 @@ -# LiteDocumentStore Implementation Checklist - -A comprehensive checklist for building a production-ready hybrid SQLite library that provides convenient JSON document storage while preserving full relational database capabilities. - ---- - -## ⚠️ Recent Unplanned Changes (Phase 1 Extensions) - -The implementation has progressed beyond the original Phase 1 scope with several architectural improvements: - -### Core Architecture Enhancements -- ✅ **Connection Ownership Pattern**: DocumentStore now supports `ownsConnection` parameter for flexible lifecycle management -- ✅ **Factory Pattern Implementation**: Added `IDocumentStoreFactory` and `DocumentStoreFactory` for proper composition -- ✅ **SqlGenerator Extraction**: SQL generation moved to separate `SqlGenerator` class for testability -- ✅ **Transaction Support**: Implemented `ExecuteInTransactionAsync` with multiple overloads - -### Configuration & Builder Pattern -- ✅ **Fluent Builder API**: Complete `DocumentStoreOptionsBuilder` with method chaining -- ✅ **Advanced Connection Options**: Added pooling config, foreign keys, busy timeout, and additional pragmas -- ✅ **Per-Store Customization**: Options can override serializer and naming conventions per instance - -### DI & Multi-Database Support -- ✅ **Keyed Services**: Full support for multiple database instances via `AddKeyedLiteDocumentStore()` -- ✅ **Factory Registration**: Automatic registration of `IDocumentStoreFactory` in DI container -- ✅ **Configurable Lifetimes**: Support for Singleton, Scoped, and Transient registrations - -### Logging & Observability -- ✅ **Comprehensive Logging**: Debug and Information level logging throughout all operations -- ✅ **NullLogger Support**: Graceful fallback when no logger is configured - -### Testing -- ✅ **Expanded Test Coverage**: Unit tests for ownership patterns, disposal, and table naming -- ✅ **Integration Tests**: Full CRUD operations, transactions with commit/rollback verification - ---- - -## 1. Core Architecture & Abstractions - -- [x] **Define interfaces for extensibility** - - [x] `IDocumentStore` - Generic document store contract (formerly `IRepository`) - - [x] ~~`IJsonSerializer`~~ - **REMOVED**: Replaced with fixed, optimized `JsonHelper` for performance - - [x] `IConnectionFactory` - Connection lifecycle management - - [x] `ITableNamingConvention` - Customizable table naming (pluralization, snake_case, etc.) - - [x] **NEW**: `IDocumentStoreFactory` - Factory for creating configured document stores - -- [x] **Configuration system** - - [x] `DocumentStoreOptions` class with builder pattern - - [x] WAL mode toggle, synchronous level, page size, cache size - - [x] Connection string builder for common scenarios - - [x] Support for in-memory databases (`:memory:`, shared cache) - - [x] **NEW**: `DocumentStoreOptionsBuilder` with fluent API - - [x] **NEW**: Factory methods for in-memory, shared in-memory, and file-based databases - - [x] **NEW**: Support for additional pragmas and custom serializer/naming convention per-store - - [x] **NEW**: Foreign keys and busy timeout configuration - -- [x] **Dependency Injection support** - - [x] `IServiceCollection.AddLiteDocumentStore()` extension method - - [x] Registers `SqliteConnection` for hybrid usage - - [x] Scoped vs Singleton connection strategies - - [x] Named/keyed store registration for multiple databases (keyed services for .NET 8+) - - [x] **NEW**: `AddKeyedLiteDocumentStore()` for managing multiple database instances - - [x] **NEW**: Automatic factory registration (`IDocumentStoreFactory`) - - [x] **NEW**: Configurable service lifetime (Singleton, Scoped, Transient) - - [x] **NEW**: Core dependencies registered as singletons (stateless services) - -- [x] **Refactor existing `Repository.cs`** - - [x] Rename to `DocumentStore` supporting `IDocumentStore` - - [x] Decouple connection lifecycle (doesn't own connection) - - [x] ~~Replace direct `System.Text.Json` calls with `IJsonSerializer`~~ - **CHANGED**: Use fixed `JsonHelper` with optimized options - - [x] Replace hardcoded `GetTableName()` with `ITableNamingConvention` - - [x] Accept `SqliteConnection` in constructor - - [x] Use `IConnectionFactory` for creation (via DI) - - [x] Add input validation (null/empty ID checks, etc.) - - [x] Add `ILogger` integration for diagnostics - - [x] Extract SQL generation to internal helper (for testability) - - [x] **NEW**: Add configurable connection ownership (`ownsConnection` parameter) - - [x] **NEW**: Implement `ExecuteInTransactionAsync` overloads for batch operations - - [x] **NEW**: Internal implementation with proper disposal pattern - - [x] **NEW**: Optimized serialization using `SerializeToUtf8Bytes` for byte[] instead of string - ---- - -## 2. Connection & Lifecycle Management - -- [x] **Connection pooling strategy** - - [x] Single long-lived connection mode (current) - -- [x] **Proper resource management** - - [x] `IAsyncDisposable` pattern (exists, verified correct) - - [x] `IDisposable` pattern for synchronous disposal - - [x] **NEW**: Conditional connection disposal based on ownership - - [x] **NEW**: Connection state validation in property accessor - - [x] Connection state validation before operations - - [x] Graceful shutdown with WAL checkpoint - -- [x] **Factory pattern implementation** - - [x] **NEW**: `IDocumentStoreFactory` interface - - [x] **NEW**: `DefaultConnectionFactory` implementation - - [x] **NEW**: `DocumentStoreFactory` for creating configured stores - - [x] **NEW**: Async factory methods (`CreateAsync`, `CreateConnectionAsync`) - - [x] **NEW**: Connection configuration via `ConfigureConnection` and `ConfigureConnectionAsync` - -- [x] **Health checks** - - [x] `IsHealthyAsync()` method for liveness probes - - [x] SQLite version validation (require 3.45+ for JSONB) - ---- - -## 3. Schema & Migration Management - -- [x] **Index management** - - [x] `CreateIndexAsync(Expression> jsonPath)` for JSON path indexes - - [x] Automatic index on `id` (already exists via PRIMARY KEY) - - [x] Composite index support via `CreateCompositeIndexAsync` - - [x] Index existence checking before creation - -- [x] **Schema versioning** - - [x] Simple migration table (`__store_migrations`) - - [x] Up/down migration support - - [x] Schema introspection helpers - - [x] `MigrationRunner` class for managing migrations independently from DocumentStore - - [x] `IMigration` interface and `Migration` base class - - [x] `SchemaIntrospector` public class for querying database schema independently - - [x] Table, column, and index introspection methods - - [x] Database statistics retrieval - - [x] Clean separation: users pass connection to MigrationRunner and SchemaIntrospector directly - ---- - -## 4. CRUD Operations Enhancement - -- [x] **Transaction Support** - - [x] **NEW**: `ExecuteInTransactionAsync` with two overloads (with/without IDbTransaction) - - [x] **NEW**: Automatic commit on success, rollback on exception - - [x] Return affected rows count from `UpsertAsync` - -- [x] **Improved Upsert** - - [x] Return affected rows count from `UpsertAsync` - - [x] Bulk upsert with single statement (`INSERT ... VALUES (...), (...), ...`) - -- [x] **Batch operations** - - [x] `UpsertManyAsync(IEnumerable<(string id, T data)> items)` - - [x] `DeleteManyAsync(IEnumerable ids)` - -- [x] **Existence checks** - - [x] `ExistsAsync(string id)` without deserializing - - [x] `CountAsync()` for table row count - ---- - -## 5. Querying Capabilities (The Hybrid Experience) - -- [x] **JSON path querying** - - [x] `QueryAsync(string jsonPath, object value)` using `json_extract()` - - [x] `QueryAsync(Expression> predicate)` with expression tree translation - - [x] Support for `$.property`, `$.nested.property`, `$.array[0]` - -- [x] **Raw SQL escape hatch** (critical for hybrid use) - - [x] **IMPLEMENTED**: Direct access to Dapper's full API via `Connection` property (exists) - -- [x] **Projection queries** - - [x] Select specific JSON fields only - - [x] `SELECT json_extract(data, '$.name') as Name FROM Customer` - - [x] `SelectAsync()` with expression-based field selection - - [x] Supports anonymous types and custom projection types - - [x] Can combine with predicates for filtered projections - ---- - -## 6. SQLite Optimizations - -- [x] **Mixed schema support** *(already supported)* - - [x] Traditional relational tables work alongside document tables via `Connection` property - - [x] Users can execute any DDL/DML using `Connection.ExecuteAsync()` - - [x] Views over JSON data can be created using raw SQL - -- [x] **Virtual columns** (SQLite generated columns) - - [x] `AddVirtualColumnAsync(Expression, columnName, createIndex)` helper - - [x] Generates `ALTER TABLE ADD COLUMN ... GENERATED ALWAYS AS (json_extract(data, '$.path'))` - - [x] Optional automatic index creation on the virtual column - - [x] Column existence check before creation - ---- - -## 7. Performance Optimizations - -- [x] **Prepared statement caching** - - [x] **INVESTIGATED & REJECTED** (January 2026) - - [x] Implemented `PreparedStatementCache` with ConcurrentDictionary for thread-safe SQL caching - - [x] Comprehensive benchmarks showed **3-5% SLOWER performance** despite 10-19% memory reduction - - [x] **Root cause**: ConcurrentDictionary lookup + cache key construction costs more than generating simple SQL strings - - [x] **Dapper already caches** at ADO.NET level (command plans, parameter mapping) - - [x] Modern C# string interpolation is highly optimized by JIT compiler - - [x] **Conclusion**: For a "Performance First" library, trading speed for ~200-400 KB per 1000 operations is the wrong tradeoff - - [x] SQL generation is fast enough; focus optimization efforts elsewhere - - [x] Full benchmark results preserved in `LiteDocumentStore.Benchmarks.PreparedStatementCacheBenchmark-report-github.md` - -- [x] **JSONB verification** - - [x] Ensure `jsonb()` function is used on write (verified in `SqlGenerator.GenerateUpsertSql` and `GenerateBulkUpsertSql`) - - [x] Ensure `json()` is used on read for deserialization (verified in all SELECT operations) - - [x] **INVESTIGATED & CONCLUDED**: Direct BLOB reading would require custom JSONB parser. Current approach using SQLite's `json()` function is optimal as System.Text.Json cannot parse SQLite's proprietary JSONB binary format. The in-database conversion is highly optimized C code. - -- [x] **Benchmarking suite** - - [x] BenchmarkDotNet project - - [x] Compare vs raw Dapper, LiteDB - - [x] Measure: single insert, bulk insert, query, full-table scan - -- [x] **Dapper type handlers** - - [x] **INVESTIGATED & REJECTED** (January 2026) - - [x] Created `SqliteJsonbTypeHandler` and `TypeHandlerRegistry` for automatic type mapping - - [x] **Root cause**: System.Text.Json cannot parse SQLite's proprietary JSONB binary format - - [x] **Required pattern**: Must use SQLite's `json()` function to convert JSONB → JSON string first - - [x] Type handlers would receive raw JSONB blobs from `SELECT data FROM table` but can't deserialize them - - [x] **Conclusion**: Manual deserialization with `JsonHelper` after `json()` conversion is the only viable approach - - [x] Current pattern (`QueryAsync` + `JsonHelper.Deserialize`) is explicit, clear, and works correctly - - [x] `SqliteJsonbTypeHandler` kept for reference but not used in the codebase - ---- - -## 8. Error Handling & Resilience - -- [x] **Custom exceptions** - - [x] **NEW**: Built-in .NET exceptions used appropriately (ArgumentNullException, ObjectDisposedException, etc.) - - [x] `LiteDocumentStoreException` base class - - [x] `TableNotFoundException`, `SerializationException`, `ConcurrencyException` - - [x] Preserve inner SQLite exceptions - -- [x] **Validation** - - [x] Validate ID is not null/empty before operations - - [x] Validate data is not null before upsert - - [x] **NEW**: Null checks with ArgumentNullException.ThrowIfNull() - ---- - -## 9. Observability - -- [x] **Logging** - - [x] `ILogger` integration - - [x] **NEW**: Debug-level logging for all operations (CreateTable, Upsert, Get, Delete, etc.) - - [x] **NEW**: Information-level logging for significant events - - [x] **NEW**: NullLogger support when no logger is provided - ---- - -## 10. Testing Infrastructure - -- [x] **Unit tests** (mock connection) - - [x] Test all public API methods - - [x] **NEW**: Connection ownership tests (ownsConnection parameter) - - [x] **NEW**: Disposal pattern verification - - [x] Edge cases: null, empty, special characters in ID - - [x] Concurrency scenarios - -- [x] **Integration tests** (real SQLite) - - [x] In-memory database for speed - - [x] **NEW**: Full CRUD operation testing - - [x] **NEW**: Transaction commit and rollback tests - - [x] **NEW**: GetAllAsync, DeleteAsync verification - - [x] File-based database for WAL testing - - [x] Multi-connection concurrency tests - -- [x] **Test helpers** - - [x] `LiteDocumentStoreTestFixture` for easy test setup - - [x] Database seeding utilities - ---- - -## 11. Documentation & Developer Experience - -- [x] **XML documentation** - - [x] All public APIs documented - - [x] **NEW**: Comprehensive XML docs for all interfaces and public classes - - [x] **NEW**: Parameter descriptions and return value documentation - -- [x] **README improvements** - - [x] Quick start guide with code examples - - [x] **NEW**: Features list with checkmarks - - [x] **NEW**: JSONB benefits explained - - [x] **NEW**: CI/CD badges - -- [x] **Executable examples** (using .NET 10 single-file execution) - - [x] QuickStart.cs - Basic CRUD operations - - [x] VirtualColumn.cs - Performance optimization with virtual columns - - [x] HybridUsage.cs - Mix document storage with traditional SQL - - [x] ProjectionQuery.cs - Field projection for performance - - [x] IndexManagement.cs - Creating and using JSON path indexes - - [x] Migration.cs - Schema versioning and evolution - - [x] TransactionBatching.cs - Batch operations example - - [x] MultiDatabase.cs - Multiple database instances with IDocumentStoreFactory - - [x] MultiDatabaseKeyed.cs - Multiple database instances with keyed services - - [x] examples/README.md - Documentation for all examples - ---- - -## 12. Packaging & Distribution - -- [x] **NuGet package** - - [x] Proper `.nuspec` or SDK-style properties - - [x] Source Link for debugging .snupkg - - [x] README in package - - [x] Icon and license metadata - -- [x] **Versioning** - - [x] Semantic versioning (follow SemVer 2.0) - - [x] Changelog maintenance (GitHub releases serve as changelog) - - [x] GitHub releases automation (via publish.yml workflow) - ---- - -## 13. Security Considerations - -- [x] **SQL injection prevention** - - [x] All user-facing table names validated/sanitized - - Table names derived from C# type names via `ITableNamingConvention` (compiler-enforced character restrictions) - - All SQL generation uses bracket notation `[tableName]` for proper escaping - - Custom naming conventions also use safe type names as input - - [x] Parameterized queries only (exists via Dapper) - - All data values use Dapper parameters (@Id, @Data, @Value, etc.) - - JSON paths for queries are embedded in SQL strings (not parameterizable), but are generated by expression tree translation or user-provided for legitimate query purposes - - [x] ~~Consider allowlist for table name characters~~ - **NOT NEEDED**: C# type name restrictions are sufficient; bracket notation provides additional safety - ---- - -## Notes - -- **Hybrid Philosophy**: The library should never prevent users from using raw SQL or traditional relational patterns. The document store features are conveniences, not constraints. -- **Performance First**: Every feature should be evaluated against the performance requirements in the README (35% faster than raw file I/O, transaction batching, async operations). -- **SQLite 3.45+ Requirement**: JSONB support is mandatory; fail fast with a clear error message on older versions. diff --git a/PERFORMANCE_INVESTIGATIONS.md b/PERFORMANCE_INVESTIGATIONS.md deleted file mode 100644 index 4172401..0000000 --- a/PERFORMANCE_INVESTIGATIONS.md +++ /dev/null @@ -1,302 +0,0 @@ -# Performance Investigations - -Based on benchmark results from ComparisonBenchmark (January 2026), this document tracks performance investigations and optimization opportunities. - -## Benchmark Results Summary - -| Operation | LiteDocumentStore | Raw Dapper | LiteDB | Overhead vs Dapper | -|-----------|------------------|------------|--------|-------------------| -| Single Insert | 7,767 ns | 5,235 ns | 7,120 ns | **+48% (2,532ns)** | -| Bulk Insert (100 docs) | 730,924 ns | 490,044 ns | 400,536 ns | **+49% (240μs)** | -| Query By ID | 2,654 ns | 2,496 ns | 2,414 ns | **+6% (158ns)** ✅ | -| Full Scan | 2,389 ns | 2,285 ns | 699 ns | **+5% (104ns)** ✅ | -| Query with Filter | 4,663 ns | 3,394 ns | 3,262 ns | **+37% (1,269ns)** | -| Delete | 1,958 ns | 1,806 ns | 498 ns | **+8% (152ns)** ✅ | - -**Memory Allocation:** -- Single Insert: 3.03 KB vs 2.24 KB (Dapper) - 35% more -- Bulk Insert: 246 KB vs 228 KB (Dapper) - 8% more ✅ -- Query with Filter: 4.20 KB vs 2.03 KB (Dapper) - 107% more - ---- - -## Investigation 1: Bulk Insert Performance Gap - -**Status**: 🔴 Not Started -**Priority**: HIGH -**Current Gap**: 49% slower than raw Dapper (240μs overhead for 100 documents = 2.4μs per doc) - -### Hypothesis - -The bulk insert implementation may have inefficiencies in: -1. SQL generation happening per batch instead of being cached -2. Unnecessary object allocations during parameter preparation -3. Transaction overhead or connection state management -4. Serialization inefficiencies - -### Investigation Steps - -- [ ] **Profile bulk insert with dotTrace or Visual Studio Profiler** - - Identify hot path and allocation sources - - Check if SQL generation is called repeatedly - -- [ ] **Review `SqlGenerator.GenerateBulkUpsertSql` implementation** - - File: [LiteDocumentStore/Core/SqlGenerator.cs](src/LiteDocumentStore/Core/SqlGenerator.cs) - - Check if string concatenation is optimal - - Verify parameter binding efficiency - -- [ ] **Review `UpsertManyAsync` implementation** - - File: [LiteDocumentStore/Core/DocumentStore.cs](src/LiteDocumentStore/Core/DocumentStore.cs) - - Check for unnecessary LINQ materializations - - Verify serialization is only happening once per document - -- [ ] **Compare raw Dapper code path vs DocumentStore** - - Add logging/tracing to identify specific overhead sources - - Measure time for: SQL generation, serialization, Dapper execution - -- [ ] **Test with prepared statement approach** - - Although rejected earlier for simple queries, bulk operations might benefit - -### Success Criteria - -- Reduce overhead to <20% (target: 600μs for 100 documents) -- Keep memory allocation delta under 10% - ---- - -## Investigation 2: Single Insert Overhead - -**Status**: 🔴 Not Started -**Priority**: MEDIUM -**Current Gap**: 48% slower than raw Dapper (2.5μs overhead per operation) - -### Hypothesis - -Single insert overhead likely comes from: -1. Method call overhead from abstraction layers -2. Table name resolution via `ITableNamingConvention` -3. SQL generation not being cached for simple operations -4. Additional allocations in the wrapper layer - -### Investigation Steps - -- [ ] **Profile single insert operation** - - Measure time spent in: naming convention, SQL generation, serialization, Dapper call - -- [ ] **Review `UpsertAsync` implementation** - - File: [LiteDocumentStore/Core/DocumentStore.cs](src/LiteDocumentStore/Core/DocumentStore.cs) - - Check parameter validation overhead - - Verify logging doesn't add significant cost - -- [ ] **Analyze memory allocations** - - Current: 3.03 KB vs 2.24 KB (790 bytes overhead) - - Identify allocation sources (strings? objects? arrays?) - -- [ ] **Consider SQL caching for common operations** - - Cache generated SQL per type (table name) - - Balance memory usage vs performance gain - -### Success Criteria - -- Reduce overhead to <25% (target: 6,500ns per insert) -- Reduce memory allocation to <2.8 KB - ---- - -## Investigation 3: Filtered Query Performance - -**Status**: 🔴 Not Started -**Priority**: HIGH -**Current Gap**: 37% slower than raw Dapper, 107% more memory - -### Hypothesis - -Expression tree translation has significant overhead: -1. `ExpressionToJsonPath` translation happening per query -2. Complex expression visitor allocations -3. Generated SQL not being cached for common patterns -4. Result materialization inefficiencies - -### Investigation Steps - -- [ ] **Profile expression tree translation** - - File: [LiteDocumentStore/Core/ExpressionToJsonPath.cs](src/LiteDocumentStore/Core/ExpressionToJsonPath.cs) - - Measure translation time vs actual query execution time - -- [ ] **Review `QueryAsync(Expression>)` implementation** - - File: [LiteDocumentStore/Core/DocumentStore.cs](src/LiteDocumentStore/Core/DocumentStore.cs) - - Check if expression compilation is happening multiple times - - Verify result materialization path - -- [ ] **Analyze memory allocations (4.20 KB vs 2.03 KB)** - - Profile to identify specific allocation sources - - Check for unnecessary LINQ operations (ToList, Select, etc.) - -- [ ] **Consider expression caching** - - Cache translated expressions using `Expression` as key - - Use `ConcurrentDictionary` for thread safety - - Measure memory vs performance tradeoff - -- [ ] **Benchmark alternative implementations** - - Direct JSON path string parameter - - Pre-compiled expression approach - -### Success Criteria - -- Reduce overhead to <20% (target: 4,000ns) -- Reduce memory allocation to <3.0 KB -- Consider offering both convenience (Expression) and performance (direct SQL) options - ---- - -## Investigation 4: Understanding LiteDB's Superior Performance - -**Status**: 🟡 Observation Only -**Priority**: LOW (informational) -**LiteDB Advantages**: 3-4x faster on full scan and delete operations - -### Observations - -LiteDB significantly outperforms SQLite for: -- **Full Table Scan**: 699ns vs 2,389ns (3.4x faster) -- **Delete**: 498ns vs 1,958ns (3.9x faster) -- **Bulk Insert**: 401μs vs 731μs (1.8x faster) - -However, LiteDB uses 2-4x more memory (10.48 KB vs 3.03 KB for single insert). - -### Investigation Steps - -- [ ] **Analyze LiteDB architecture** - - Understand in-memory data structure optimizations - - Review how it handles sequential access patterns - - Check if results are "pre-deserialized" in memory - -- [ ] **Consider hybrid approach** - - Could DocumentStore detect sequential scan patterns? - - Is there an intermediate caching layer that would help? - -- [ ] **Document tradeoffs** - - LiteDB trades memory for speed - - SQLite/JSONB provides durability and relational capabilities - - Make tradeoff explicit in documentation - -### Outcome - -Document when users should prefer: -- **LiteDocumentStore**: Hybrid relational + document, ACID transactions, SQL access -- **LiteDB**: Pure document store, in-memory speed, simpler use cases - ---- - -## Investigation 5: Memory Allocation Patterns - -**Status**: 🔴 Not Started -**Priority**: MEDIUM - -### Current Allocations vs Raw Dapper - -| Operation | LiteDocumentStore | Raw Dapper | Overhead | -|-----------|------------------|------------|----------| -| Single Insert | 3.03 KB | 2.24 KB | +35% | -| Bulk Insert | 246 KB | 228 KB | +8% ✅ | -| Query By ID | 1.63 KB | 1.41 KB | +16% | -| Query Filtered | 4.20 KB | 2.03 KB | +107% 🔴 | -| Full Scan | 1.58 KB | 1.41 KB | +12% | -| Delete | 1.21 KB | 1.03 KB | +17% | - -### Investigation Steps - -- [ ] **Run benchmarks with allocation profiler** - - Use BenchmarkDotNet's `[MemoryDiagnoser(displayGenColumns: true)]` - - Identify Gen0/Gen1/Gen2 collection patterns - -- [ ] **Review object pooling opportunities** - - `JsonSerializerOptions` pooling - - StringBuilder pooling for SQL generation - - Byte array pooling for serialization buffers - -- [ ] **Analyze string allocations** - - Table name strings - - SQL query strings - - JSON path strings - -- [ ] **Consider struct over class** - - Internal DTOs that don't need reference semantics - - Parameter objects for Dapper - -### Success Criteria - -- Keep allocation overhead under 20% for all operations -- Eliminate allocations in hot paths where possible - ---- - -## Investigation 6: Virtual Column Index Performance Validation - -**Status**: 🟢 Already Benchmarked (VirtualColumnBenchmark.cs) -**Priority**: LOW (validation only) - -### Existing Results - -Virtual column benchmarks already demonstrate significant benefits of indexing JSON fields. Review existing benchmark results to document: - -- [ ] Performance improvement with indexes (expected: 10-100x for large datasets) -- [ ] Memory tradeoffs -- [ ] When to recommend virtual columns to users - ---- - -## Optimization Priority Matrix - -| Investigation | Impact | Effort | Priority | -|---------------|--------|--------|----------| -| #1 Bulk Insert | High | Medium | **HIGH** ⭐ | -| #3 Filtered Query | High | Medium-High | **HIGH** ⭐ | -| #2 Single Insert | Medium | Low-Medium | **MEDIUM** | -| #5 Memory Patterns | Medium | Medium | **MEDIUM** | -| #4 LiteDB Comparison | Low | Low | **LOW** | -| #6 Virtual Columns | Low | Low | **LOW** | - ---- - -## Performance Goals - -### Phase 1: Quick Wins (Target: 1-2 weeks) -- Identify and eliminate obvious allocation sources -- Add SQL caching for common operations -- Profile hot paths to find low-hanging fruit - -**Target**: Reduce write overhead to 25-30%, maintain <10% read overhead - -### Phase 2: Deep Optimization (Target: 1 month) -- Optimize expression tree translation -- Implement smart caching strategies -- Consider object pooling for frequently allocated objects - -**Target**: Reduce write overhead to 15-20%, maintain <10% read overhead - -### Phase 3: Advanced Techniques (Target: 2-3 months) -- Investigate async I/O patterns -- Consider source generators for compile-time SQL generation -- Explore zero-allocation serialization paths - -**Target**: Approach raw Dapper performance (5-10% overhead) - ---- - -## Notes - -- Always measure before and after optimizations -- Don't optimize prematurely - profile first -- Document any tradeoffs (memory vs speed, API convenience vs performance) -- Consider offering both "fast path" and "convenient path" APIs for power users -- The `Connection` property already provides an escape hatch for performance-critical code - ---- - -## Tracking - -**Created**: January 11, 2026 -**Last Updated**: January 11, 2026 -**Benchmark Version**: ComparisonBenchmark v1.0 -**Next Review**: After Investigation #1 and #3 are complete diff --git a/README.md b/README.md index cf621f1..96fd18f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # LiteDocumentStore -[![CI](https://github.com/idotta/jsonb-store/actions/workflows/ci.yml/badge.svg)](https://github.com/idotta/jsonb-store/actions/workflows/ci.yml) -[![Code Quality](https://github.com/idotta/jsonb-store/actions/workflows/code-quality.yml/badge.svg)](https://github.com/idotta/jsonb-store/actions/workflows/code-quality.yml) +[![CI](https://github.com/idotta/lite-doc-store/actions/workflows/ci.yml/badge.svg)](https://github.com/idotta/lite-doc-store/actions/workflows/ci.yml) +[![Code Quality](https://github.com/idotta/lite-doc-store/actions/workflows/code-quality.yml/badge.svg)](https://github.com/idotta/lite-doc-store/actions/workflows/code-quality.yml) [![NuGet](https://img.shields.io/nuget/v/LiteDocumentStore.svg)](https://www.nuget.org/packages/LiteDocumentStore/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -A high-performance, single-file application data format using C#, SQLite (Microsoft.Data.Sqlite), and Dapper. +A high-performance, single-file application data format using C# and SQLite (Microsoft.Data.Sqlite). Objects are stored in SQLite's binary **JSONB** format and the store is Native-AOT / trim compatible. # Core Architecture @@ -16,7 +16,7 @@ A single SQLite .db file acting as an "Application File Format". Treat SQLite as a hybrid relational/document store. JSON data is stored in **JSONB format** (binary JSON introduced in SQLite 3.45+) for optimal storage efficiency and query performance. ## Data Access Layer -Use Dapper for next-to-zero mapping overhead. +Raw ADO.NET over `Microsoft.Data.Sqlite` — parameters bound explicitly, results read by ordinal (no ORM, no runtime reflection or IL generation), so the library stays Native-AOT / trim safe. ## Custom Logic Automatic JSON serialization/deserialization of C# objects into SQLite BLOB columns using JSONB format. @@ -46,7 +46,7 @@ dotnet build ## Features -- ✅ **Generic Repository Pattern**: Type-safe CRUD operations with automatic table naming +- ✅ **Document Store API** (`IDocumentStore`): Type-safe CRUD operations with automatic table naming - ✅ **Async/Await**: All database operations are fully async - ✅ **JSONB Format**: Uses SQLite 3.45+ JSONB for binary-optimized JSON storage - ✅ **Virtual Columns**: Index JSON properties for up to 1,300x faster queries @@ -55,13 +55,48 @@ dotnet build - ✅ **Zero SQL Injection Risk**: Table names derived from types, not user input - ✅ **Cross-Platform**: Works on Windows, Linux, and macOS - ✅ **.NET 10**: Built on the latest .NET platform +- ✅ **Native-AOT / Trim Compatible**: `true`; serialization goes through `System.Text.Json` `JsonTypeInfo` - ✅ **Comprehensive Tests**: Unit and integration tests with xUnit +## Quick Start + +Register the store through dependency injection and resolve `IDocumentStore`: + +```csharp +using LiteDocumentStore; +using Microsoft.Extensions.DependencyInjection; + +var services = new ServiceCollection(); +services.AddLiteDocumentStore(options => +{ + options.ConnectionString = "Data Source=app.db"; + options.EnableWalMode = true; + // For Native-AOT, supply source-generated metadata: + // options.SerializerOptions = new JsonSerializerOptions { TypeInfoResolver = MyJsonContext.Default }; +}); + +await using var provider = services.BuildServiceProvider(); +var store = provider.GetRequiredService(); + +await store.CreateTableAsync(); +await store.UpsertAsync("c1", new Customer { Name = "Ada", Email = "ada@example.com" }); + +var customer = await store.GetAsync("c1"); + +// Query documents by JSON path + value +var byName = await store.QueryAsync("$.Name", "Ada"); + +// For ranges, joins, or virtual-column seeks, drop to raw SQL via the escape hatch: +var conn = store.Connection; // Microsoft.Data.Sqlite SqliteConnection +``` + +Without DI, build the store via `IDocumentStoreFactory.CreateAsync(DocumentStoreOptions)`. + ## Dependencies - .NET 10 -- Dapper - Microsoft.Data.Sqlite +- Microsoft.Extensions.DependencyInjection.Abstractions / Logging.Abstractions ## JSONB Storage Benefits diff --git a/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.ComparisonBenchmark-report-github.md b/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.ComparisonBenchmark-report-github.md deleted file mode 100644 index 63d4e74..0000000 --- a/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.ComparisonBenchmark-report-github.md +++ /dev/null @@ -1,42 +0,0 @@ -``` - -BenchmarkDotNet v0.15.8, Windows 11 (10.0.26100.7462/24H2/2024Update/HudsonValley) -13th Gen Intel Core i7-13650HX 2.60GHz, 1 CPU, 20 logical and 14 physical cores -.NET SDK 10.0.101 - [Host] : .NET 10.0.1 (10.0.1, 10.0.125.57005), X64 RyuJIT x86-64-v3 - Job-MNMNNY : .NET 10.0.1 (10.0.1, 10.0.125.57005), X64 RyuJIT x86-64-v3 - -IterationCount=15 RunStrategy=Throughput - -``` -| Method | index | id | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | -|---------------------------------- |------ |----------- |-------------:|------------:|------------:|------:|--------:|--------:|-------:|-----------:|------------:| -| **LiteDocumentStore_BulkInsert** | **?** | **?** | **686,321.8 ns** | **4,551.78 ns** | **4,035.03 ns** | **?** | **?** | **16.6016** | **2.9297** | **209.78 KB** | **?** | -| RawDapper_BulkInsert | ? | ? | 419,683.4 ns | 2,960.65 ns | 2,769.39 ns | ? | ? | 18.5547 | - | 227.5 KB | ? | -| LiteDB_BulkInsert | ? | ? | 391,443.4 ns | 5,071.03 ns | 4,495.34 ns | ? | ? | 89.8438 | 3.9063 | 1102.69 KB | ? | -| LiteDocumentStore_FullScan | ? | ? | 2,388.9 ns | 23.58 ns | 22.06 ns | ? | ? | 0.1259 | - | 1.58 KB | ? | -| RawDapper_FullScan | ? | ? | 2,247.5 ns | 31.11 ns | 29.10 ns | ? | ? | 0.1144 | - | 1.41 KB | ? | -| LiteDB_FullScan | ? | ? | 670.7 ns | 8.81 ns | 7.81 ns | ? | ? | 0.1926 | - | 2.37 KB | ? | -| LiteDocumentStore_QueryByCategory | ? | ? | 4,172.4 ns | 88.06 ns | 82.37 ns | ? | ? | 0.3204 | - | 3.95 KB | ? | -| RawDapper_QueryByCategory | ? | ? | 3,229.7 ns | 34.58 ns | 32.34 ns | ? | ? | 0.1640 | - | 2.03 KB | ? | -| LiteDB_QueryByCategory | ? | ? | 3,275.6 ns | 87.99 ns | 82.31 ns | ? | ? | 0.6409 | - | 7.95 KB | ? | -| | | | | | | | | | | | | -| **LiteDocumentStore_SingleInsert** | **0** | **?** | **7,459.4 ns** | **50.09 ns** | **41.83 ns** | **1.00** | **0.01** | **0.2213** | **-** | **2.78 KB** | **1.00** | -| RawDapper_SingleInsert | 0 | ? | 4,784.1 ns | 37.26 ns | 34.86 ns | 0.64 | 0.01 | 0.1755 | - | 2.24 KB | 0.81 | -| LiteDB_SingleInsert | 0 | ? | 6,728.8 ns | 89.57 ns | 83.78 ns | 0.90 | 0.01 | 0.8545 | 0.1526 | 10.48 KB | 3.77 | -| | | | | | | | | | | | | -| **LiteDocumentStore_SingleInsert** | **50** | **?** | **7,527.6 ns** | **44.81 ns** | **39.73 ns** | **1.00** | **0.01** | **0.2136** | **-** | **2.79 KB** | **1.00** | -| RawDapper_SingleInsert | 50 | ? | 4,830.5 ns | 44.14 ns | 39.13 ns | 0.64 | 0.01 | 0.1831 | - | 2.26 KB | 0.81 | -| LiteDB_SingleInsert | 50 | ? | 6,752.9 ns | 58.36 ns | 48.73 ns | 0.90 | 0.01 | 0.8545 | 0.1526 | 10.54 KB | 3.78 | -| | | | | | | | | | | | | -| **LiteDocumentStore_QueryById** | **?** | **doc-000000** | **2,545.9 ns** | **31.70 ns** | **29.65 ns** | **?** | **?** | **0.1297** | **-** | **1.63 KB** | **?** | -| RawDapper_QueryById | ? | doc-000000 | 2,499.2 ns | 31.19 ns | 27.65 ns | ? | ? | 0.1144 | - | 1.41 KB | ? | -| LiteDB_QueryById | ? | doc-000000 | 2,409.0 ns | 26.12 ns | 20.39 ns | ? | ? | 0.4921 | 0.0038 | 6.06 KB | ? | -| | | | | | | | | | | | | -| **LiteDocumentStore_QueryById** | **?** | **doc-000050** | **2,569.0 ns** | **51.31 ns** | **47.99 ns** | **?** | **?** | **0.1297** | **-** | **1.63 KB** | **?** | -| RawDapper_QueryById | ? | doc-000050 | 2,439.4 ns | 36.86 ns | 32.68 ns | ? | ? | 0.1144 | - | 1.41 KB | ? | -| LiteDB_QueryById | ? | doc-000050 | 2,367.8 ns | 31.54 ns | 27.96 ns | ? | ? | 0.4807 | 0.0038 | 5.9 KB | ? | -| | | | | | | | | | | | | -| **LiteDocumentStore_Delete** | **?** | **doc-000099** | **1,924.7 ns** | **31.88 ns** | **29.82 ns** | **?** | **?** | **0.0954** | **-** | **1.21 KB** | **?** | -| RawDapper_Delete | ? | doc-000099 | 1,774.4 ns | 8.32 ns | 7.78 ns | ? | ? | 0.0839 | - | 1.03 KB | ? | -| LiteDB_Delete | ? | doc-000099 | 498.8 ns | 3.40 ns | 3.01 ns | ? | ? | 0.2041 | 0.0010 | 2.51 KB | ? | diff --git a/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.ProjectionQueryBenchmark-report-github.md b/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.ProjectionQueryBenchmark-report-github.md deleted file mode 100644 index 5bfac92..0000000 --- a/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.ProjectionQueryBenchmark-report-github.md +++ /dev/null @@ -1,19 +0,0 @@ -``` - -BenchmarkDotNet v0.15.8, Windows 11 (10.0.26100.7462/24H2/2024Update/HudsonValley) -13th Gen Intel Core i7-13650HX 2.60GHz, 1 CPU, 20 logical and 14 physical cores -.NET SDK 10.0.101 - [Host] : .NET 10.0.1 (10.0.1, 10.0.125.57005), X64 RyuJIT x86-64-v3 - Job-MNMNNY : .NET 10.0.1 (10.0.1, 10.0.125.57005), X64 RyuJIT x86-64-v3 - -IterationCount=15 RunStrategy=Throughput - -``` -| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | -|---------------------------------------------- |-----------:|----------:|----------:|------:|--------:|---------:|---------:|-----------:|------------:| -| 'Full document retrieval (baseline)' | 5,281.0 μs | 109.61 μs | 102.53 μs | 1.00 | 0.03 | 648.4375 | 640.6250 | 7946.77 KB | 1.000 | -| 'Projection query - 2 fields' | 547.1 μs | 2.57 μs | 2.40 μs | 0.10 | 0.00 | 10.7422 | 2.9297 | 137.38 KB | 0.017 | -| 'Projection query - 4 fields' | 1,019.0 μs | 8.17 μs | 7.64 μs | 0.19 | 0.00 | 21.4844 | 9.7656 | 264.49 KB | 0.033 | -| 'Projection query - nested field' | 885.8 μs | 3.64 μs | 3.23 μs | 0.17 | 0.00 | 15.6250 | 3.9063 | 201.64 KB | 0.025 | -| 'Projection query with filter - 2 fields' | 212.7 μs | 1.20 μs | 1.00 μs | 0.04 | 0.00 | 1.4648 | - | 20.27 KB | 0.003 | -| 'Full documents with filter (for comparison)' | 656.6 μs | 9.29 μs | 8.69 μs | 0.12 | 0.00 | 64.4531 | 21.4844 | 798.25 KB | 0.100 | diff --git a/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.SimplifiedComparisonBenchmark-report-github.md b/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.SimplifiedComparisonBenchmark-report-github.md deleted file mode 100644 index 9cd7b70..0000000 --- a/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.SimplifiedComparisonBenchmark-report-github.md +++ /dev/null @@ -1,29 +0,0 @@ -``` - -BenchmarkDotNet v0.15.8, Windows 11 (10.0.26100.7462/24H2/2024Update/HudsonValley) -13th Gen Intel Core i7-13650HX 2.60GHz, 1 CPU, 20 logical and 14 physical cores -.NET SDK 10.0.101 - [Host] : .NET 10.0.1 (10.0.1, 10.0.125.57005), X64 RyuJIT x86-64-v3 - Job-INMAZI : .NET 10.0.1 (10.0.1, 10.0.125.57005), X64 RyuJIT x86-64-v3 - -IterationCount=5 RunStrategy=Throughput WarmupCount=3 - -``` -| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | -|---------------------------------- |-----------:|-----------:|----------:|------:|--------:|-------:|-------:|----------:|------------:| -| LiteDocumentStore_SingleInsert | 5.201 μs | 0.1807 μs | 0.0280 μs | 1.00 | 0.01 | 0.1984 | - | 2.45 KB | 1.00 | -| RawDapper_SingleInsert | 4.917 μs | 0.0835 μs | 0.0129 μs | 0.95 | 0.01 | 0.1755 | - | 2.24 KB | 0.92 | -| LiteDocumentStore_BulkInsert | 213.521 μs | 4.6604 μs | 1.2103 μs | 41.06 | 0.29 | 8.0566 | 0.9766 | 99.79 KB | 40.81 | -| RawDapper_BulkInsert | 218.645 μs | 27.3825 μs | 7.1111 μs | 42.04 | 1.27 | 9.2773 | - | 114.36 KB | 46.77 | -| LiteDocumentStore_QueryById | 2.762 μs | 0.2620 μs | 0.0680 μs | 0.53 | 0.01 | 0.1297 | - | 1.59 KB | 0.65 | -| RawDapper_QueryById | 2.514 μs | 0.0365 μs | 0.0095 μs | 0.48 | 0.00 | 0.1144 | - | 1.41 KB | 0.58 | -| LiteDocumentStore_FullScan | 2.426 μs | 0.3003 μs | 0.0780 μs | 0.47 | 0.01 | 0.1183 | - | 1.48 KB | 0.61 | -| RawDapper_FullScan | 2.245 μs | 0.0310 μs | 0.0081 μs | 0.43 | 0.00 | 0.1144 | - | 1.41 KB | 0.58 | -| LiteDocumentStore_QueryByCategory | 4.104 μs | 0.1166 μs | 0.0303 μs | 0.79 | 0.01 | 0.3052 | - | 3.84 KB | 1.57 | -| RawDapper_QueryByCategory | 3.436 μs | 0.4574 μs | 0.0708 μs | 0.66 | 0.01 | 0.1640 | - | 2.03 KB | 0.83 | -| LiteDocumentStore_Delete | 1.920 μs | 0.0763 μs | 0.0118 μs | 0.37 | 0.00 | 0.0954 | - | 1.17 KB | 0.48 | -| RawDapper_Delete | 1.822 μs | 0.0400 μs | 0.0104 μs | 0.35 | 0.00 | 0.0839 | - | 1.03 KB | 0.42 | -| LiteDocumentStore_BulkDelete | 9.402 μs | 1.5436 μs | 0.2389 μs | 1.81 | 0.04 | 0.6104 | - | 7.59 KB | 3.10 | -| RawDapper_BulkDelete | 8.158 μs | 0.2743 μs | 0.0712 μs | 1.57 | 0.01 | 0.5798 | 0.1068 | 7.15 KB | 2.92 | -| LiteDocumentStore_Update | 5.149 μs | 0.0857 μs | 0.0223 μs | 0.99 | 0.01 | 0.1984 | - | 2.45 KB | 1.00 | -| RawDapper_Update | 4.789 μs | 0.0220 μs | 0.0034 μs | 0.92 | 0.00 | 0.1831 | - | 2.26 KB | 0.92 | diff --git a/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.VirtualColumnBenchmark-report-github.md b/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.VirtualColumnBenchmark-report-github.md deleted file mode 100644 index e4fe101..0000000 --- a/benchmarks/2026-01-11/LiteDocumentStore.Benchmarks.VirtualColumnBenchmark-report-github.md +++ /dev/null @@ -1,28 +0,0 @@ -``` - -BenchmarkDotNet v0.15.8, Windows 11 (10.0.26100.7462/24H2/2024Update/HudsonValley) -13th Gen Intel Core i7-13650HX 2.60GHz, 1 CPU, 20 logical and 14 physical cores -.NET SDK 10.0.101 - [Host] : .NET 10.0.1 (10.0.1, 10.0.125.57005), X64 RyuJIT x86-64-v3 - Job-MNMNNY : .NET 10.0.1 (10.0.1, 10.0.125.57005), X64 RyuJIT x86-64-v3 - -IterationCount=15 RunStrategy=Throughput - -``` -| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Gen2 | Allocated | Alloc Ratio | -|------------------------------------------------------ |---------------:|--------------:|--------------:|-------:|--------:|----------:|----------:|----------:|------------:|------------:| -| 'Query by category WITHOUT virtual column' | 11,891.610 μs | 113.2174 μs | 94.5416 μs | 1.000 | 0.01 | 125.0000 | 93.7500 | - | 1536.58 KB | 1.000 | -| 'Query by category WITH virtual column and index' | 3,798.686 μs | 67.2992 μs | 52.5428 μs | 0.319 | 0.00 | 125.0000 | 93.7500 | - | 1536.39 KB | 1.000 | -| 'Query by price WITHOUT virtual column' | 10,135.764 μs | 52.8025 μs | 44.0925 μs | 0.852 | 0.01 | - | - | - | 3.95 KB | 0.003 | -| 'Query by price WITH virtual column and index' | 146,248.902 μs | 3,178.2773 μs | 2,972.9626 μs | 12.299 | 0.26 | 4333.3333 | 3333.3333 | 1000.0000 | 45429.96 KB | 29.566 | -| 'Query by SKU WITHOUT virtual column' | 10,541.055 μs | 327.1313 μs | 305.9988 μs | 0.886 | 0.03 | - | - | - | 5.31 KB | 0.003 | -| 'Query by SKU WITH virtual column and index' | 8.255 μs | 0.0749 μs | 0.0700 μs | 0.001 | 0.00 | 0.4120 | - | - | 5.13 KB | 0.003 | -| 'Query nested property WITHOUT virtual column' | 13,749.342 μs | 264.7320 μs | 247.6305 μs | 1.156 | 0.02 | 62.5000 | 31.2500 | - | 765.73 KB | 0.498 | -| 'Query nested property WITH virtual column and index' | 1,946.694 μs | 84.5957 μs | 74.9918 μs | 0.164 | 0.01 | 58.5938 | 15.6250 | - | 765.51 KB | 0.498 | -| 'Raw SQL: Category query (indexed)' | 2,502.182 μs | 34.6057 μs | 32.3702 μs | 0.210 | 0.00 | 15.6250 | 3.9063 | - | 212.68 KB | 0.138 | -| 'Raw SQL: Category query (no index)' | 10,542.902 μs | 237.8290 μs | 198.5980 μs | 0.887 | 0.02 | 15.6250 | - | - | 212.7 KB | 0.138 | -| 'Raw SQL: Price query (indexed)' | 76,107.038 μs | 1,523.1085 μs | 1,424.7167 μs | 6.400 | 0.13 | 571.4286 | 428.5714 | 142.8571 | 6425.51 KB | 4.182 | -| 'Raw SQL: Price query (no index)' | 19,386.379 μs | 382.9179 μs | 358.1816 μs | 1.630 | 0.03 | 500.0000 | 468.7500 | 218.7500 | 6425.14 KB | 4.181 | -| 'Raw SQL: SKU query (indexed)' | 5.060 μs | 0.1089 μs | 0.1019 μs | 0.000 | 0.00 | 0.1068 | - | - | 1.37 KB | 0.001 | -| 'Raw SQL: SKU query (no index)' | 10,297.303 μs | 66.0207 μs | 51.5446 μs | 0.866 | 0.01 | - | - | - | 1.39 KB | 0.001 | -| 'Add virtual column (column creation overhead)' | 47,832.566 μs | 915.3973 μs | 714.6820 μs | 4.023 | 0.07 | 1000.0000 | 1000.0000 | 1000.0000 | 3004.09 KB | 1.955 | diff --git a/examples/AotVerification.cs b/examples/AotVerification.cs new file mode 100644 index 0000000..37521e5 --- /dev/null +++ b/examples/AotVerification.cs @@ -0,0 +1,78 @@ +#!/usr/bin/env dotnet run +// AOT Verification Example - Prove the library works under Native AOT +// +// Run (JIT): dotnet run AotVerification.cs +// Publish (AOT): dotnet publish AotVerification.cs -r win-x64 (use your RID) +// +// This example supplies a source-generated JsonSerializerContext so that NO reflection-based +// JSON serialization is used. It exercises the full surviving document-store surface end to end. + +#:package Microsoft.Extensions.DependencyInjection@10.0.1 + +#:project ../src/LiteDocumentStore/LiteDocumentStore.csproj + +#:property PublishAot=true + +using System.Text.Json; +using System.Text.Json.Serialization; +using LiteDocumentStore; +using Microsoft.Extensions.DependencyInjection; + +// Back the store with a source-generated context - the AOT-safe serialization path. +var serializerOptions = new JsonSerializerOptions +{ + TypeInfoResolver = AppJsonContext.Default +}; + +var options = new DocumentStoreOptionsBuilder() + .UseInMemory() + .WithSerializerOptions(serializerOptions) + .Build(); + +var services = new ServiceCollection(); +services.AddLiteDocumentStore(options); +using var provider = services.BuildServiceProvider(); +var store = provider.GetRequiredService(); + +// Schema +await store.CreateTableAsync(); + +// Writes: single + bulk +await store.UpsertAsync("p1", new Person("p1", "Ada Lovelace", "ada@example.com", 36)); +await store.UpsertManyAsync( +[ + ("p2", new Person("p2", "Alan Turing", "alan@example.com", 41)), + ("p3", new Person("p3", "Grace Hopper", "grace@example.com", 85)), +]); + +// Reads +var ada = await store.GetAsync("p1"); +Console.WriteLine($"Get p1 => {ada?.Name}"); + +var all = (await store.GetAllAsync()).ToList(); +Console.WriteLine($"GetAll => {all.Count} people"); + +// Index + query by JSON path (equality) +await store.CreateIndexAsync(p => p.Email); +var byEmail = (await store.QueryAsync("$.Email", "grace@example.com")).ToList(); +Console.WriteLine($"Query $.Email => {byEmail.Count} ({byEmail.FirstOrDefault()?.Name})"); + +// Aggregates +Console.WriteLine($"Count => {await store.CountAsync()}"); +Console.WriteLine($"Exists p2 => {await store.ExistsAsync("p2")}"); + +// Delete +Console.WriteLine($"Delete p3 => {await store.DeleteAsync("p3")}"); +Console.WriteLine($"Count after delete => {await store.CountAsync()}"); + +// Health +Console.WriteLine($"Healthy => {await store.IsHealthyAsync()}"); + +Console.WriteLine("\n✓ AOT verification completed - all operations ran with source-generated JSON (no reflection)."); + +// Model + source-generated serialization context +record Person(string Id, string Name, string Email, int Age); + +[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(Person))] +internal partial class AppJsonContext : JsonSerializerContext; diff --git a/examples/ProjectionQuery.cs b/examples/ProjectionQuery.cs deleted file mode 100644 index 72de415..0000000 --- a/examples/ProjectionQuery.cs +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/env dotnet run -// Projection Query Example - Select only needed fields for better performance -// -// Run this example with: dotnet run ProjectionQuery.cs -// -// Projection queries extract specific JSON fields instead of deserializing entire documents. -// This reduces memory usage and improves performance, especially with large or nested objects. - -#:package Dapper@2.1.66 -#:package Microsoft.Extensions.DependencyInjection@10.0.1 -#:package Microsoft.Extensions.Logging@10.0.1 -#:package Microsoft.Extensions.Logging.Console@10.0.1 - -#:project ../src/LiteDocumentStore/LiteDocumentStore.csproj - -#:property PublishAot=false - -using Dapper; -using System.Diagnostics; -using LiteDocumentStore; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -// Enable reflection-based JSON serialization for .NET 10+ -AppContext.SetSwitch("System.Text.Json.JsonSerializer.IsReflectionEnabledByDefault", true); - -var services = new ServiceCollection(); -services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Information)); - -var serviceProvider = services.BuildServiceProvider(); -var logger = serviceProvider.GetRequiredService>(); - -// Create store -logger.LogInformation("Creating customer database..."); -var options = new DocumentStoreOptionsBuilder() - .UseInMemory() - .WithWalMode(false) - .Build(); - -services.AddLiteDocumentStore(options); -serviceProvider = services.BuildServiceProvider(); -var store = serviceProvider.GetRequiredService(); - -// Create table and seed data -await store.CreateTableAsync(); - -logger.LogInformation("Seeding customers with rich nested data..."); -await store.ExecuteInTransactionAsync(async () => -{ - for (int i = 1; i <= 1000; i++) - { - await store.UpsertAsync( - $"c{i}", - new Customer( - $"c{i}", - $"Customer {i}", - $"customer{i}@example.com", - 25 + (i % 50), - new Address( - $"{i} Main Street", - i % 10 == 0 ? "New York" : i % 10 == 1 ? "Los Angeles" : "Chicago", - "NY", - $"{10000 + i}", - "USA" - ), - new ContactInfo( - $"555-{i:D4}", - $"555-{i + 1000:D4}", - $"555-{i + 2000:D4}", - $"555-{i + 3000:D4}" - ), - new Preferences( - EmailNotifications: i % 2 == 0, - SmsNotifications: i % 3 == 0, - Language: "en-US", - Timezone: "America/New_York" - ), - new OrderHistory( - OrderIds: Enumerable.Range(1, i % 20).Select(o => $"order-{i}-{o}").ToList(), - TotalSpent: 100m * (i % 10), - LastOrderDate: DateTime.UtcNow.AddDays(-i % 365) - ), - Metadata: new Dictionary - { - ["source"] = "web", - ["campaign"] = $"campaign-{i % 5}", - ["tags"] = string.Join(",", Enumerable.Range(1, 5).Select(t => $"tag{t}")) - } - ) - ); - } -}); - -Console.WriteLine($"Inserted {await store.CountAsync()} customers"); - -// Benchmark 1: GetAllAsync - retrieves ENTIRE documents -logger.LogInformation("\nBenchmark 1: GetAllAsync (full deserialization)..."); -var sw = Stopwatch.StartNew(); -var allCustomers = (await store.GetAllAsync()).ToList(); -sw.Stop(); -Console.WriteLine($"Retrieved {allCustomers.Count} full customers in {sw.ElapsedMilliseconds}ms"); -Console.WriteLine($"Memory: Each customer includes Address, Contact, Preferences, OrderHistory, Metadata"); - -// Benchmark 2: SelectAsync - only Name and Email -logger.LogInformation("\nBenchmark 2: SelectAsync for CustomerSummary (Name, Email only)..."); -sw.Restart(); -var summaries = await store.SelectAsync( - c => new CustomerSummary { Name = c.Name, Email = c.Email } -); -sw.Stop(); -var summariesList = summaries.ToList(); -Console.WriteLine($"Retrieved {summariesList.Count} summaries in {sw.ElapsedMilliseconds}ms"); -Console.WriteLine($"Sample: {summariesList[0].Name} ({summariesList[0].Email})"); - -// Benchmark 3: SelectAsync with nested properties -logger.LogInformation("\nBenchmark 3: SelectAsync with nested properties (Name, City, State)..."); -sw.Restart(); -var locations = (await store.SelectAsync( - c => new CustomerLocation { Name = c.Name, City = c.Address.City, State = c.Address.State } -)).ToList(); -sw.Stop(); -Console.WriteLine($"Retrieved {locations.Count} locations in {sw.ElapsedMilliseconds}ms"); -var nyCustomers = locations.Count(l => l.City == "New York"); -Console.WriteLine($"Customers in New York: {nyCustomers}"); - -// Benchmark 4: SelectAsync with predicate (filtered projection) -logger.LogInformation("\nBenchmark 4: SelectAsync with predicate (New York customers only)..."); -sw.Restart(); -var nyLocations = (await store.SelectAsync( - c => c.Address.City == "New York", - c => new CustomerLocation { Name = c.Name, City = c.Address.City, State = c.Address.State } -)).ToList(); -sw.Stop(); -Console.WriteLine($"Retrieved {nyLocations.Count} New York customers in {sw.ElapsedMilliseconds}ms"); - -// Benchmark 5: SelectAsync with multiple nested paths -logger.LogInformation("\nBenchmark 5: SelectAsync with deep nesting (Name, Email, Phone)..."); -sw.Restart(); -var contacts = await store.SelectAsync( - c => new CustomerContact { Name = c.Name, Email = c.Email, PrimaryPhone = c.Contact.PrimaryPhone } -); -sw.Stop(); -var contactsList = contacts.ToList(); -Console.WriteLine($"Retrieved {contactsList.Count} contacts in {sw.ElapsedMilliseconds}ms"); -Console.WriteLine($"Sample: {contactsList[0].Name} - {contactsList[0].PrimaryPhone}"); - -// Demonstrate raw SQL for complex projections -logger.LogInformation("\nBonus: Raw SQL for complex aggregations..."); -var query = @" - SELECT - json_extract(data, '$.Address.City') as City, - COUNT(*) as CustomerCount, - AVG(CAST(json_extract(data, '$.Age') as INTEGER)) as AvgAge - FROM Customer - GROUP BY json_extract(data, '$.Address.City') - ORDER BY CustomerCount DESC"; - -var cityStats = await store.Connection.QueryAsync<(string City, int CustomerCount, double AvgAge)>(query); -Console.WriteLine("\nCustomers by City:"); -foreach (var (city, count, avgAge) in cityStats) -{ - Console.WriteLine($" {city}: {count} customers (avg age: {avgAge:F1})"); -} - -Console.WriteLine("\n✓ Projection Query example completed!"); -Console.WriteLine("\nKey Takeaways:"); -Console.WriteLine(" • Use SelectAsync to project specific fields"); -Console.WriteLine(" • Reduces deserialization overhead and memory usage"); -Console.WriteLine(" • Supports nested property access (c.Address.City)"); -Console.WriteLine(" • Can combine with predicates for filtered projections"); -Console.WriteLine(" • For complex aggregations, use raw SQL with json_extract()"); - -// Define models -record Address(string Street, string City, string State, string ZipCode, string Country); -record ContactInfo(string PrimaryPhone, string SecondaryPhone, string Fax, string Mobile); -record Preferences(bool EmailNotifications, bool SmsNotifications, string Language, string Timezone); -record OrderHistory(List OrderIds, decimal TotalSpent, DateTime LastOrderDate); - -record Customer( - string Id, - string Name, - string Email, - int Age, - Address Address, - ContactInfo Contact, - Preferences Preferences, - OrderHistory OrderHistory, - Dictionary Metadata -); - -// Projection DTOs -record CustomerSummary -{ - public string Name { get; init; } = null!; - public string Email { get; init; } = null!; -} - -record CustomerLocation -{ - public string Name { get; init; } = null!; - public string City { get; init; } = null!; - public string State { get; init; } = null!; -} - -record CustomerContact -{ - public string Name { get; init; } = null!; - public string Email { get; init; } = null!; - public string PrimaryPhone { get; init; } = null!; -} diff --git a/examples/README.md b/examples/README.md index eb18e7a..6b672b3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -88,38 +88,13 @@ Demonstrates that LiteDocumentStore is a HYBRID library - you get convenient doc **Perfect for**: Real-world applications needing both flexible schemas and relational queries -**Key concept**: The `Connection` property gives you full Dapper access. You're never locked into just the document API - drop down to SQL anytime. +**Key concept**: The `Connection` property exposes the raw `SqliteConnection` for full ADO.NET access. You're never locked into just the document API - drop down to SQL anytime. **Run time**: < 1 second --- -### 4. [ProjectionQuery.cs](ProjectionQuery.cs) -**Select only needed fields** - Reduce memory and improve performance - -Demonstrates how to extract specific JSON fields instead of deserializing entire documents: -- Seeding 1,000 customers with rich nested data (Address, Contact, Preferences, OrderHistory, Metadata) -- Benchmarking `GetAllAsync()` (full document deserialization) -- Using `SelectAsync()` for field projection -- Projecting simple properties (Name, Email only) -- Projecting nested properties (Address.City, Address.State) -- Filtered projections with predicates -- Complex aggregations with raw SQL - -**Memory & Performance**: -- Full documents: All nested objects deserialized -- Projection: Only specified fields extracted from JSON -- Reduces memory footprint for list views and reports - -**Perfect for**: List views, reports, APIs returning summaries, reducing bandwidth - -**Key concept**: Projection uses `json_extract()` in SQL to retrieve only needed fields, avoiding full deserialization overhead. - -**Run time**: ~1-2 seconds - ---- - -### 5. [IndexManagement.cs](IndexManagement.cs) +### 4. [IndexManagement.cs](IndexManagement.cs) **Optimize query performance with JSON indexes** - Avoid full table scans Shows how to create indexes on JSON properties for dramatic query speedups: @@ -147,7 +122,7 @@ Shows how to create indexes on JSON properties for dramatic query speedups: --- -### 6. [Migration.cs](Migration.cs) +### 5. [Migration.cs](Migration.cs) **Schema versioning and evolution** - Track and apply schema changes Demonstrates a complete migration system for managing schema changes over time: @@ -176,7 +151,7 @@ Demonstrates a complete migration system for managing schema changes over time: --- -### 7. [TransactionBatching.cs](TransactionBatching.cs) +### 6. [TransactionBatching.cs](TransactionBatching.cs) **Maximize performance with transactions** - 50x-100x speedup for bulk operations Demonstrates how transaction batching dramatically improves performance for bulk operations: @@ -200,7 +175,7 @@ Demonstrates how transaction batching dramatically improves performance for bulk --- -### 8. [MultiDatabase.cs](MultiDatabase.cs) +### 7. [MultiDatabase.cs](MultiDatabase.cs) **Multiple databases with factory pattern** - Manage separate databases for tenants or domains Demonstrates using `IDocumentStoreFactory` to create and manage multiple independent database instances: @@ -224,7 +199,7 @@ Demonstrates using `IDocumentStoreFactory` to create and manage multiple indepen --- -### 9. [MultiDatabaseKeyed.cs](MultiDatabaseKeyed.cs) +### 8. [MultiDatabaseKeyed.cs](MultiDatabaseKeyed.cs) **Multiple databases with keyed DI** - Type-safe dependency injection for multiple databases Demonstrates using keyed services (requires .NET 8+) to register and inject multiple database instances: @@ -249,6 +224,32 @@ Demonstrates using keyed services (requires .NET 8+) to register and inject mult --- +### 9. [AotVerification.cs](AotVerification.cs) +**Native AOT compatibility** - Reflection-free JSON with a source-generated context + +Proves the library runs under Native AOT / trimming by supplying a source-generated +`JsonSerializerContext` and exercising the full document-store surface with no reflection-based +serialization: +- Defining a `[JsonSerializable]` partial `JsonSerializerContext` +- Wiring it in via `DocumentStoreOptionsBuilder.WithSerializerOptions(...)` +- Running CRUD, bulk writes, `CreateIndexAsync`, `QueryAsync(jsonPath, value)`, + `CountAsync`/`ExistsAsync`, and `IsHealthyAsync` +- Sets `#:property PublishAot=true` so it can be AOT-published directly + +**Run (JIT)**: `dotnet run AotVerification.cs` + +**Publish (AOT)**: `dotnet publish AotVerification.cs -r ` (e.g. `win-x64`) + +**Perfect for**: Verifying AOT builds, learning the source-generated serialization setup + +**Key concept**: For AOT, set `SerializerOptions` to options backed by a source-generated +context (`new JsonSerializerOptions { TypeInfoResolver = MyContext.Default }`). When no options +are supplied, the store falls back to reflection-based serialization (non-AOT only). + +**Run time**: < 1 second + +--- + ## Example Structure Each example follows this pattern: @@ -284,13 +285,13 @@ Each example follows this pattern: |---------|-----------|--------------|----------|---------| | QuickStart.cs | CRUD basics | 6 records | <1s | `UpsertAsync`, `GetAsync`, `DeleteAsync` | | TransactionBatching.cs | Bulk performance | 1K orders | ~5-10s | `ExecuteInTransactionAsync`, `UpsertManyAsync` | -| VirtualColumn.cs | Query performance | 10K products | ~2-3s | `AddVirtualColumnAsync`, `QueryAsync` | +| VirtualColumn.cs | Query performance | 10K products | ~2-3s | `AddVirtualColumnAsync`, `QueryAsync`, raw SQL | | HybridUsage.cs | SQL integration | Small | <1s | `Connection`, raw SQL | -| ProjectionQuery.cs | Field selection | 1K customers | ~1-2s | `SelectAsync` | | IndexManagement.cs | Indexing | 5K customers | ~1-2s | `CreateIndexAsync`, `CreateCompositeIndexAsync` | | Migration.cs | Schema versioning | Small | ~1-2s | `MigrationRunner`, `SchemaIntrospector` | | MultiDatabase.cs | Factory pattern | Small | <1s | `IDocumentStoreFactory.Create` | | MultiDatabaseKeyed.cs | Keyed DI services | Small | ~1s | `AddKeyedLiteDocumentStore`, `[FromKeyedServices]` | +| AotVerification.cs | Native AOT | 3 records | <1s | `WithSerializerOptions`, source-gen context | ## Feedback diff --git a/examples/VirtualColumn.cs b/examples/VirtualColumn.cs index 4066f9c..de23d8f 100644 --- a/examples/VirtualColumn.cs +++ b/examples/VirtualColumn.cs @@ -75,10 +75,20 @@ await store.UpsertAsync( Console.WriteLine($"Inserted {await store.CountAsync()} products"); -// Benchmark: Query WITHOUT virtual column +// Helper: run a raw SQL WHERE clause against the Product table and deserialize the documents. +// Raw SQL over store.Connection is the hybrid escape hatch - it lets queries reference the +// indexed virtual columns directly (json_extract, used by QueryAsync, cannot use those indexes). +async Task> QueryByRawWhereAsync(string whereClause) +{ + var sql = $"SELECT json(data) as JsonData FROM Product WHERE {whereClause}"; + var rows = await store.Connection.QueryAsync(sql); + return rows.Select(j => System.Text.Json.JsonSerializer.Deserialize(j)!).ToList(); +} + +// Benchmark: Query WITHOUT virtual column (json_extract full scan via the string-path API) logger.LogInformation("\nBenchmark 1: Query by Category WITHOUT virtual column..."); var sw = Stopwatch.StartNew(); -var electronicsWithout = (await store.QueryAsync(p => p.Category == "Electronics")).ToList(); +var electronicsWithout = (await store.QueryAsync("$.Category", "Electronics")).ToList(); sw.Stop(); Console.WriteLine($"Found {electronicsWithout.Count} products in {sw.ElapsedMilliseconds}ms"); Console.WriteLine($"Sample: {electronicsWithout[0].Name} (SKU: {electronicsWithout[0].Sku})"); @@ -86,7 +96,7 @@ await store.UpsertAsync( // Benchmark: Query by SKU WITHOUT virtual column logger.LogInformation("\nBenchmark 2: Query by SKU WITHOUT virtual column..."); sw.Restart(); -var productWithout = (await store.QueryAsync(p => p.Sku == "SKU-005000")).ToList(); +var productWithout = (await store.QueryAsync("$.Sku", "SKU-005000")).ToList(); sw.Stop(); Console.WriteLine($"Found {productWithout.Count} product(s) in {sw.ElapsedMilliseconds}ms"); @@ -94,13 +104,14 @@ await store.UpsertAsync( logger.LogInformation("\nAdding virtual columns with indexes..."); await store.AddVirtualColumnAsync(p => p.Category, "category", createIndex: true); await store.AddVirtualColumnAsync(p => p.Sku, "sku", createIndex: true); -await store.AddVirtualColumnAsync(p => p.Price, "price", createIndex: true); +// Use REAL so numeric range comparisons ([price] > 100) work correctly (TEXT affinity would compare as strings). +await store.AddVirtualColumnAsync(p => p.Price, "price", createIndex: true, columnType: "REAL"); Console.WriteLine("Virtual columns created: [category], [sku], [price]"); -// Benchmark: Query WITH virtual column +// Benchmark: Query WITH virtual column (raw SQL hits the [category] index) logger.LogInformation("\nBenchmark 3: Query by Category WITH virtual column..."); sw.Restart(); -var electronicsWith = (await store.QueryAsync(p => p.Category == "Electronics")).ToList(); +var electronicsWith = await QueryByRawWhereAsync("[category] = 'Electronics'"); sw.Stop(); Console.WriteLine($"Found {electronicsWith.Count} products in {sw.ElapsedMilliseconds}ms"); var improvement1 = sw.ElapsedMilliseconds > 0 ? "faster" : "nearly instant"; @@ -109,7 +120,7 @@ await store.UpsertAsync( // Benchmark: Query by SKU WITH virtual column logger.LogInformation("\nBenchmark 4: Query by SKU WITH virtual column (indexed)..."); sw.Restart(); -var productWith = (await store.QueryAsync(p => p.Sku == "SKU-005000")).ToList(); +var productWith = await QueryByRawWhereAsync("[sku] = 'SKU-005000'"); sw.Stop(); Console.WriteLine($"Found {productWith.Count} product(s) in {sw.ElapsedMilliseconds}ms"); Console.WriteLine($"Result: Should be sub-millisecond (index seek)!"); @@ -117,7 +128,7 @@ await store.UpsertAsync( // Demonstrate range queries on indexed columns logger.LogInformation("\nBenchmark 5: Range query on Price (indexed virtual column)..."); sw.Restart(); -var expensiveProducts = (await store.QueryAsync(p => p.Price > 100)).ToList(); +var expensiveProducts = await QueryByRawWhereAsync("[price] > 100"); sw.Stop(); Console.WriteLine($"Found {expensiveProducts.Count} products over $100 in {sw.ElapsedMilliseconds}ms"); @@ -149,7 +160,7 @@ AND [sku] LIKE 'SKU-00%' Console.WriteLine(" • Virtual columns extract JSON fields into indexed columns"); Console.WriteLine(" • Point queries (exact match) get 100x-1000x speedup"); Console.WriteLine(" • Range queries and complex filters also benefit"); -Console.WriteLine(" • QueryAsync automatically uses virtual columns when available"); +Console.WriteLine(" • Reference virtual columns via raw SQL ([col]) to use their indexes"); Console.WriteLine(" • Best for frequently queried fields in large tables"); // Define models diff --git a/src/LiteDocumentStore/Core/DocumentStore.cs b/src/LiteDocumentStore/Core/DocumentStore.cs index 36a831a..b0566ef 100644 --- a/src/LiteDocumentStore/Core/DocumentStore.cs +++ b/src/LiteDocumentStore/Core/DocumentStore.cs @@ -1,5 +1,5 @@ using System.Data; -using Dapper; +using System.Text.Json; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -8,15 +8,15 @@ namespace LiteDocumentStore; /// /// A high-performance document store for storing JSON objects in SQLite. -/// Uses Dapper for minimal mapping overhead and supports JSON document storage using JSONB format (SQLite 3.45+). -/// Can optionally own and manage the lifecycle of its SqliteConnection. +/// Uses raw ADO.NET (Microsoft.Data.Sqlite) with explicit parameter binding and JSONB +/// storage (SQLite 3.45+). Can optionally own and manage the lifecycle of its SqliteConnection. /// internal sealed class DocumentStore : IDocumentStore { private readonly SqliteConnection _connection; private readonly ITableNamingConvention _tableNamingConvention; private readonly ILogger _logger; - private readonly VirtualColumnCache _virtualColumnCache; + private readonly JsonSerializerOptions _serializerOptions; private readonly bool _ownsConnection; private bool _disposed; @@ -27,16 +27,21 @@ internal sealed class DocumentStore : IDocumentStore /// Table naming convention (defaults to DefaultTableNamingConvention) /// Logger for diagnostics (optional) /// Whether this store owns and should dispose the connection (default: false) + /// + /// JSON serializer options. For AOT, back these with a source-generated JsonSerializerContext. + /// When null, a reflection-based fallback is used (non-AOT only). + /// public DocumentStore( SqliteConnection connection, ITableNamingConvention? tableNamingConvention = null, ILogger? logger = null, - bool ownsConnection = false) + bool ownsConnection = false, + JsonSerializerOptions? serializerOptions = null) { _connection = connection ?? throw new ArgumentNullException(nameof(connection)); _tableNamingConvention = tableNamingConvention ?? new DefaultTableNamingConvention(); _logger = logger ?? NullLogger.Instance; - _virtualColumnCache = new VirtualColumnCache(connection); + _serializerOptions = serializerOptions ?? JsonHelper.CreateDefaultReflectionOptions(); _ownsConnection = ownsConnection; } @@ -80,7 +85,7 @@ public async Task CreateTableAsync() var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateCreateTableSql(tableName); - await _connection.ExecuteAsync(sql); + await _connection.ExecuteAsync(sql).ConfigureAwait(false); } /// @@ -97,16 +102,10 @@ public async Task UpsertAsync(string id, T data) ArgumentNullException.ThrowIfNull(data); var tableName = _tableNamingConvention.GetTableName(); - var jsonBytes = JsonHelper.SerializeToUtf8Bytes(data); + var jsonBytes = JsonHelper.SerializeToUtf8Bytes(data, _serializerOptions); var sql = SqlGenerator.GenerateUpsertSql(tableName); - var affectedRows = await _connection.ExecuteAsync(sql, new - { - Id = id, - Data = jsonBytes - }); - - return affectedRows; + return await _connection.ExecuteAsync(sql, ("Id", id), ("Data", jsonBytes)).ConfigureAwait(false); } /// @@ -127,8 +126,7 @@ public async Task UpsertManyAsync(IEnumerable<(string id, T data)> items var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateBulkUpsertSql(tableName, itemsList.Count); - // Build dynamic parameters object - var parameters = new DynamicParameters(); + var parameters = new (string, object?)[itemsList.Count * 2]; for (int i = 0; i < itemsList.Count; i++) { // Validate all items @@ -142,14 +140,12 @@ public async Task UpsertManyAsync(IEnumerable<(string id, T data)> items } var (id, data) = itemsList[i]; - var jsonBytes = JsonHelper.SerializeToUtf8Bytes(data); - parameters.Add($"Id{i}", id); - parameters.Add($"Data{i}", jsonBytes); + var jsonBytes = JsonHelper.SerializeToUtf8Bytes(data, _serializerOptions); + parameters[i * 2] = ($"Id{i}", id); + parameters[(i * 2) + 1] = ($"Data{i}", jsonBytes); } - var affectedRows = await _connection.ExecuteAsync(sql, parameters); - - return affectedRows; + return await _connection.ExecuteAsync(sql, parameters).ConfigureAwait(false); } /// @@ -166,7 +162,7 @@ public async Task UpsertManyAsync(IEnumerable<(string id, T data)> items var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateGetByIdSql(tableName); - var json = await _connection.QueryFirstOrDefaultAsync(sql, new { Id = id }); + var json = await _connection.QueryFirstStringAsync(sql, ("Id", id)).ConfigureAwait(false); if (string.IsNullOrEmpty(json)) { @@ -174,7 +170,7 @@ public async Task UpsertManyAsync(IEnumerable<(string id, T data)> items return default; } - return JsonHelper.Deserialize(json); + return JsonHelper.Deserialize(json, _serializerOptions); } /// @@ -186,19 +182,8 @@ public async Task> GetAllAsync() var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateGetAllSql(tableName); - var jsonResults = await _connection.QueryAsync(sql); - - var results = new List(); - foreach (var json in jsonResults) - { - var item = JsonHelper.Deserialize(json); - if (item != null) - { - results.Add(item); - } - } - - return results; + var jsonResults = await _connection.QueryStringsAsync(sql).ConfigureAwait(false); + return DeserializeResults(jsonResults); } /// @@ -215,7 +200,7 @@ public async Task DeleteAsync(string id) var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateDeleteSql(tableName); - var affectedRows = await _connection.ExecuteAsync(sql, new { Id = id }); + var affectedRows = await _connection.ExecuteAsync(sql, ("Id", id)).ConfigureAwait(false); var deleted = affectedRows > 0; if (!deleted) @@ -253,16 +238,13 @@ public async Task DeleteManyAsync(IEnumerable ids) var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateBulkDeleteSql(tableName, idsList.Count); - // Build dynamic parameters object - var parameters = new DynamicParameters(); + var parameters = new (string, object?)[idsList.Count]; for (int i = 0; i < idsList.Count; i++) { - parameters.Add($"Id{i}", idsList[i]); + parameters[i] = ($"Id{i}", idsList[i]); } - var affectedRows = await _connection.ExecuteAsync(sql, parameters); - - return affectedRows; + return await _connection.ExecuteAsync(sql, parameters).ConfigureAwait(false); } /// @@ -279,7 +261,7 @@ public async Task ExistsAsync(string id) var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateExistsSql(tableName); - return await _connection.ExecuteScalarAsync(sql, new { Id = id }); + return await _connection.ExecuteScalarAsync(sql, ("Id", id)).ConfigureAwait(false); } /// @@ -291,7 +273,7 @@ public async Task CountAsync() var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateCountSql(tableName); - return await _connection.ExecuteScalarAsync(sql); + return await _connection.ExecuteScalarAsync(sql).ConfigureAwait(false); } /// @@ -310,48 +292,21 @@ public async Task> QueryAsync(string jsonPath, TValue var tableName = _tableNamingConvention.GetTableName(); var sql = SqlGenerator.GenerateQueryByJsonPathSql(tableName, jsonPath); - var jsonResults = await _connection.QueryAsync(sql, new { Value = value }).ConfigureAwait(false); - var documents = DeserializeResults(jsonResults); - return documents; - } - - /// - public async Task> QueryAsync(System.Linq.Expressions.Expression> predicate) - { - ObjectDisposedException.ThrowIf(_disposed, this); - EnsureConnectionOpen(); - - ArgumentNullException.ThrowIfNull(predicate); - - var tableName = _tableNamingConvention.GetTableName(); - - // Get virtual columns for this table to enable index usage - var virtualColumns = await _virtualColumnCache.GetAsync(tableName).ConfigureAwait(false); - - // Translate the expression to SQL WHERE clause - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - var sql = SqlGenerator.GenerateQueryWithWhereSql(tableName, whereClause); - - var jsonResults = await _connection.QueryAsync(sql, parameters).ConfigureAwait(false); - var documents = DeserializeResults(jsonResults); - - return documents; + var jsonResults = await _connection.QueryStringsAsync(sql, ("Value", value)).ConfigureAwait(false); + return DeserializeResults(jsonResults); } /// /// Deserializes JSON results to a list of typed objects. /// Uses a single-pass loop to avoid LINQ overhead and multiple enumerator allocations. /// - private static List DeserializeResults(IEnumerable jsonResults) + private List DeserializeResults(IReadOnlyCollection jsonResults) { - // Pre-size list if we can determine count without enumeration - var results = jsonResults is ICollection collection - ? new List(collection.Count) - : []; + var results = new List(jsonResults.Count); foreach (var json in jsonResults) { - if (JsonHelper.Deserialize(json) is { } item) + if (JsonHelper.Deserialize(json, _serializerOptions) is { } item) { results.Add(item); } @@ -363,13 +318,13 @@ private static List DeserializeResults(IEnumerable jsonResults) /// public async Task ExecuteInTransactionAsync(Func action) { - await ExecuteInTransactionCoreAsync(action); + await ExecuteInTransactionCoreAsync(action).ConfigureAwait(false); } /// public async Task ExecuteInTransactionAsync(Func action) { - await ExecuteInTransactionCoreAsync(_ => action()); + await ExecuteInTransactionCoreAsync(_ => action()).ConfigureAwait(false); } /// @@ -380,14 +335,10 @@ private async Task ExecuteInTransactionCoreAsync(Func acti ObjectDisposedException.ThrowIf(_disposed, this); EnsureConnectionOpen(); - // Use existing transaction if any? - // _connection.BeginTransaction() requires the connection to be open. - // It throws if a transaction is already active on this connection (SQLite supports one transaction per connection unless using Savepoints). - // Since we don't control the connection, we should check if we can start a transaction. - // However, standard ADO.NET SqliteConnection.BeginTransaction() will fail if currently in a transaction. - // For now, naive implementation: try to begin. - // Ideally we should support nested transactions or check, but simpler first. - + // BeginTransaction requires the connection to be open and throws if a transaction is + // already active (SQLite supports one transaction per connection). Commands created via + // SqliteConnection.CreateCommand automatically enlist in the active transaction, so the + // document operations invoked inside the action participate without extra wiring. using var transaction = _connection.BeginTransaction(); try { @@ -413,9 +364,9 @@ public async Task CreateIndexAsync(System.Linq.Expressions.Expression( + var indexExists = await _connection.ExecuteScalarAsync( SqlGenerator.GenerateCheckIndexExistsSql(), - new { IndexName = finalIndexName }).ConfigureAwait(false); + ("IndexName", finalIndexName)).ConfigureAwait(false); if (indexExists > 0) { @@ -439,13 +390,13 @@ public async Task CreateCompositeIndexAsync(System.Linq.Expressions.Expressio EnsureConnectionOpen(); var tableName = _tableNamingConvention.GetTableName(); - var pathStrings = jsonPaths.Select(ExtractJsonPath); + var pathStrings = jsonPaths.Select(ExtractJsonPath).ToList(); var finalIndexName = indexName ?? GenerateCompositeIndexName(tableName, pathStrings); // Check if index already exists - var indexExists = await _connection.QueryFirstOrDefaultAsync( + var indexExists = await _connection.ExecuteScalarAsync( SqlGenerator.GenerateCheckIndexExistsSql(), - new { IndexName = finalIndexName }).ConfigureAwait(false); + ("IndexName", finalIndexName)).ConfigureAwait(false); if (indexExists > 0) { @@ -461,6 +412,8 @@ public async Task CreateCompositeIndexAsync(System.Linq.Expressions.Expressio /// Extracts the JSON path from a lambda expression. /// Supports simple property access (e.g., x => x.Email) and nested properties (e.g., x => x.Address.City). /// Uses property names as-is to match the default System.Text.Json serialization (PascalCase). + /// Only reads member names from the expression tree (no compilation or closure evaluation), + /// so it is AOT/trim safe. /// private static string ExtractJsonPath(System.Linq.Expressions.Expression> expression) { @@ -489,21 +442,7 @@ private static string ExtractJsonPath(System.Linq.Expressions.Expression - /// Converts a property name to camelCase for JSON path. - /// - private static string ToCamelCase(string str) - { - if (string.IsNullOrEmpty(str) || char.IsLower(str[0])) - { - return str; - } - - return char.ToLowerInvariant(str[0]) + str[1..]; + return "$." + string.Join(".", members); } /// @@ -560,18 +499,15 @@ public async Task AddVirtualColumnAsync( await _connection.ExecuteAsync(addColumnSql).ConfigureAwait(false); } - // Register the virtual column in cache (whether newly created or already existing) - _virtualColumnCache.Register(tableName, new VirtualColumnInfo(pathString, columnName, columnType)); - // Create index on the virtual column if requested if (createIndex) { var indexName = $"idx_{tableName}_{columnName}"; // Check if index already exists - var indexExists = await _connection.QueryFirstOrDefaultAsync( + var indexExists = await _connection.ExecuteScalarAsync( SqlGenerator.GenerateCheckIndexExistsSql(), - new { IndexName = indexName }).ConfigureAwait(false); + ("IndexName", indexName)).ConfigureAwait(false); if (indexExists > 0) { @@ -585,48 +521,6 @@ public async Task AddVirtualColumnAsync( } } - /// - public async Task> SelectAsync( - System.Linq.Expressions.Expression> selector) - { - ObjectDisposedException.ThrowIf(_disposed, this); - EnsureConnectionOpen(); - - ArgumentNullException.ThrowIfNull(selector); - - var tableName = _tableNamingConvention.GetTableName(); - var fieldSelections = ExpressionToJsonPath.ExtractFieldSelections(selector); - var sql = SqlGenerator.GenerateSelectFieldsSql(tableName, fieldSelections); - - var results = await _connection.QueryAsync(sql).ConfigureAwait(false); - return results; - } - - /// - public async Task> SelectAsync( - System.Linq.Expressions.Expression> predicate, - System.Linq.Expressions.Expression> selector) - { - ObjectDisposedException.ThrowIf(_disposed, this); - EnsureConnectionOpen(); - - ArgumentNullException.ThrowIfNull(predicate); - ArgumentNullException.ThrowIfNull(selector); - - var tableName = _tableNamingConvention.GetTableName(); - var fieldSelections = ExpressionToJsonPath.ExtractFieldSelections(selector); - - // Get virtual columns for this table to enable index usage - var virtualColumns = await _virtualColumnCache.GetAsync(tableName).ConfigureAwait(false); - - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - var sql = SqlGenerator.GenerateSelectFieldsWithWhereSql(tableName, fieldSelections, whereClause); - - var results = await _connection.QueryAsync(sql, parameters).ConfigureAwait(false); - - return results; - } - /// public async Task IsHealthyAsync() { @@ -647,7 +541,7 @@ public async Task IsHealthyAsync() } // Verify SQLite version supports JSONB (3.45+) - var versionString = await _connection.QueryFirstOrDefaultAsync( + var versionString = await _connection.QueryFirstStringAsync( "SELECT sqlite_version()").ConfigureAwait(false); if (string.IsNullOrWhiteSpace(versionString)) @@ -672,7 +566,7 @@ public async Task IsHealthyAsync() } // Test basic query execution - await _connection.QueryFirstOrDefaultAsync("SELECT 1").ConfigureAwait(false); + await _connection.ExecuteScalarAsync("SELECT 1").ConfigureAwait(false); _logger.LogDebug("Health check passed: SQLite version {Version}", version); return true; @@ -737,7 +631,7 @@ private async Task PerformWalCheckpointAsync() if (_connection.State == ConnectionState.Open) { // Check if we're in WAL mode - var journalMode = await _connection.QueryFirstOrDefaultAsync( + var journalMode = await _connection.QueryFirstStringAsync( "PRAGMA journal_mode").ConfigureAwait(false); if (string.Equals(journalMode, "wal", StringComparison.OrdinalIgnoreCase)) @@ -768,7 +662,7 @@ private void PerformWalCheckpoint() if (_connection.State == ConnectionState.Open) { // Check if we're in WAL mode - var journalMode = _connection.QueryFirstOrDefault("PRAGMA journal_mode"); + var journalMode = _connection.QueryFirstString("PRAGMA journal_mode"); if (string.Equals(journalMode, "wal", StringComparison.OrdinalIgnoreCase)) { diff --git a/src/LiteDocumentStore/Core/DocumentStoreOptions.cs b/src/LiteDocumentStore/Core/DocumentStoreOptions.cs index a5fe67d..1d5086f 100644 --- a/src/LiteDocumentStore/Core/DocumentStoreOptions.cs +++ b/src/LiteDocumentStore/Core/DocumentStoreOptions.cs @@ -1,3 +1,6 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + namespace LiteDocumentStore; /// @@ -64,6 +67,16 @@ public sealed class DocumentStoreOptions /// public List AdditionalPragmas { get; set; } = []; + /// + /// Gets or sets the used to (de)serialize documents. + /// For Native AOT / trimming, set this to options backed by a source-generated + /// , e.g. + /// new JsonSerializerOptions { TypeInfoResolver = MyContext.Default }. + /// When null (the default), the store falls back to reflection-based serialization, + /// which works only in non-AOT scenarios. + /// + public JsonSerializerOptions? SerializerOptions { get; set; } + /// /// Creates a new instance of DocumentStoreOptions with default settings. /// @@ -148,6 +161,12 @@ public static DocumentStoreOptions ForSharedInMemory(string cacheName = "shared" /// /// Creates a copy of the current options. /// + /// + /// is shared by reference intentionally: the instance carries + /// the source-generated TypeInfoResolver and its metadata cache, which must be shared for + /// AOT correctness and performance. System.Text.Json also makes a + /// read-only after its first use, so the shared instance is effectively immutable in practice. + /// /// A new DocumentStoreOptions instance with copied values public DocumentStoreOptions Clone() { @@ -161,7 +180,8 @@ public DocumentStoreOptions Clone() BusyTimeoutMs = BusyTimeoutMs, EnableForeignKeys = EnableForeignKeys, TableNamingConvention = TableNamingConvention, - AdditionalPragmas = [.. AdditionalPragmas] + AdditionalPragmas = [.. AdditionalPragmas], + SerializerOptions = SerializerOptions }; } } diff --git a/src/LiteDocumentStore/Core/DocumentStoreOptionsBuilder.cs b/src/LiteDocumentStore/Core/DocumentStoreOptionsBuilder.cs index 0a93876..088738b 100644 --- a/src/LiteDocumentStore/Core/DocumentStoreOptionsBuilder.cs +++ b/src/LiteDocumentStore/Core/DocumentStoreOptionsBuilder.cs @@ -1,3 +1,5 @@ +using System.Text.Json; + namespace LiteDocumentStore; /// @@ -169,6 +171,20 @@ public DocumentStoreOptionsBuilder WithTableNamingConvention(ITableNamingConvent return this; } + /// + /// Sets the used to (de)serialize documents. + /// For Native AOT / trimming, back these options with a source-generated + /// , e.g. + /// new JsonSerializerOptions { TypeInfoResolver = MyContext.Default }. + /// + /// The serializer options to use + /// This builder for method chaining + public DocumentStoreOptionsBuilder WithSerializerOptions(JsonSerializerOptions serializerOptions) + { + _options.SerializerOptions = serializerOptions; + return this; + } + /// /// Adds a custom PRAGMA statement to execute on connection open. /// diff --git a/src/LiteDocumentStore/Core/ExpressionToJsonPath.cs b/src/LiteDocumentStore/Core/ExpressionToJsonPath.cs deleted file mode 100644 index bde14b0..0000000 --- a/src/LiteDocumentStore/Core/ExpressionToJsonPath.cs +++ /dev/null @@ -1,493 +0,0 @@ -using System.Linq.Expressions; -using System.Reflection; -using System.Text; - -namespace LiteDocumentStore; - -/// -/// Translates LINQ expression trees to SQLite JSON path syntax for use with json_extract(). -/// Supports simple property access, nested properties, and array indexing. -/// -internal static class ExpressionToJsonPath -{ - /// - /// Converts a LINQ expression to a JSON path string. - /// Supports patterns like: $.property, $.nested.property, $.array[0] - /// - /// The type being queried - /// The expression to translate - /// A JSON path string starting with $. - /// Thrown when the expression cannot be translated - public static string Translate(Expression> expression) - { - ArgumentNullException.ThrowIfNull(expression); - - var body = expression.Body; - - // Handle boxing conversions (e.g., value types to object) - if (body is UnaryExpression { NodeType: ExpressionType.Convert } unary) - { - body = unary.Operand; - } - - var path = new StringBuilder("$"); - BuildPath(body, path); - return path.ToString(); - } - - /// - /// Converts a predicate expression to SQL WHERE conditions and extracts parameter values. - /// Supports equality comparisons on JSON properties. - /// When virtual columns are provided, generates optimized SQL using column references instead of json_extract. - /// - /// The type being queried - /// The predicate expression to translate - /// Optional dictionary of virtual columns keyed by JSON path - /// A tuple containing the WHERE clause and a dictionary of parameter values - /// Thrown when the predicate cannot be translated - public static (string whereClause, Dictionary parameters) TranslatePredicate( - Expression> predicate, - IReadOnlyDictionary? virtualColumns = null) - { - ArgumentNullException.ThrowIfNull(predicate); - - var parameters = new Dictionary(); - var whereClause = BuildWhereClause(predicate.Body, parameters, virtualColumns); - - return (whereClause, parameters); - } - - private static void BuildPath(Expression expression, StringBuilder path) - { - switch (expression) - { - case MemberExpression memberExpr: - // Recursively build the path for nested properties - if (memberExpr.Expression != null && memberExpr.Expression.NodeType != ExpressionType.Parameter) - { - BuildPath(memberExpr.Expression, path); - } - // Use the property name as-is (PascalCase) to match System.Text.Json default behavior - path.Append('.').Append(memberExpr.Member.Name); - break; - - case MethodCallExpression methodCall when IsIndexerAccess(methodCall): - // Handle array/list indexing: list[0] - BuildPath(methodCall.Object!, path); - var index = GetIndexValue(methodCall.Arguments[0]); - path.Append('[').Append(index).Append(']'); - break; - - case BinaryExpression binaryExpr when binaryExpr.NodeType == ExpressionType.ArrayIndex: - // Handle array indexing: array[0] - BuildPath(binaryExpr.Left, path); - var arrayIndex = GetIndexValue(binaryExpr.Right); - path.Append('[').Append(arrayIndex).Append(']'); - break; - - case ParameterExpression: - // Base case - do nothing, we already have "$" - break; - - default: - throw new NotSupportedException( - $"Expression type '{expression.NodeType}' is not supported for JSON path translation. " + - $"Supported patterns: $.property, $.nested.property, $.array[0]"); - } - } - - private static string BuildWhereClause( - Expression expression, - Dictionary parameters, - IReadOnlyDictionary? virtualColumns) - { - switch (expression) - { - case BinaryExpression binary when binary.NodeType == ExpressionType.Equal: - return BuildEqualityComparison(binary, parameters, virtualColumns); - - case BinaryExpression binary when binary.NodeType == ExpressionType.NotEqual: - return BuildInequalityComparison(binary, parameters, virtualColumns); - - case BinaryExpression binary when binary.NodeType == ExpressionType.AndAlso: - var left = BuildWhereClause(binary.Left, parameters, virtualColumns); - var right = BuildWhereClause(binary.Right, parameters, virtualColumns); - return $"({left} AND {right})"; - - case BinaryExpression binary when binary.NodeType == ExpressionType.OrElse: - var leftOr = BuildWhereClause(binary.Left, parameters, virtualColumns); - var rightOr = BuildWhereClause(binary.Right, parameters, virtualColumns); - return $"({leftOr} OR {rightOr})"; - - case BinaryExpression binary when IsComparisonOperator(binary.NodeType): - return BuildComparisonOperator(binary, parameters, virtualColumns); - - case MethodCallExpression methodCall: - return BuildMethodCallCondition(methodCall, parameters, virtualColumns); - - default: - throw new NotSupportedException( - $"Expression type '{expression.NodeType}' is not supported for WHERE clause translation. " + - $"Supported: ==, !=, &&, ||, >, <, >=, <=, string methods"); - } - } - - private static string BuildEqualityComparison( - BinaryExpression binary, - Dictionary parameters, - IReadOnlyDictionary? virtualColumns) - { - var (jsonPath, value) = ExtractComparisonParts(binary); - var paramName = $"p{parameters.Count}"; - parameters[paramName] = value; - - var columnRef = GetColumnReference(jsonPath, virtualColumns); - return $"{columnRef} = @{paramName}"; - } - - private static string BuildInequalityComparison( - BinaryExpression binary, - Dictionary parameters, - IReadOnlyDictionary? virtualColumns) - { - var (jsonPath, value) = ExtractComparisonParts(binary); - var paramName = $"p{parameters.Count}"; - parameters[paramName] = value; - - var columnRef = GetColumnReference(jsonPath, virtualColumns); - return $"{columnRef} != @{paramName}"; - } - - private static string BuildComparisonOperator( - BinaryExpression binary, - Dictionary parameters, - IReadOnlyDictionary? virtualColumns) - { - var (jsonPath, value) = ExtractComparisonParts(binary); - var paramName = $"p{parameters.Count}"; - parameters[paramName] = value; - - var op = binary.NodeType switch - { - ExpressionType.GreaterThan => ">", - ExpressionType.GreaterThanOrEqual => ">=", - ExpressionType.LessThan => "<", - ExpressionType.LessThanOrEqual => "<=", - _ => throw new NotSupportedException($"Comparison operator {binary.NodeType} not supported") - }; - - var columnRef = GetColumnReference(jsonPath, virtualColumns); - return $"{columnRef} {op} @{paramName}"; - } - - private static string BuildMethodCallCondition( - MethodCallExpression methodCall, - Dictionary parameters, - IReadOnlyDictionary? virtualColumns) - { - // Support common string methods like Contains, StartsWith, EndsWith - if (methodCall.Method.DeclaringType == typeof(string)) - { - var jsonPath = TranslateToJsonPath(methodCall.Object!); - var value = GetConstantValue(methodCall.Arguments[0]); - var paramName = $"p{parameters.Count}"; - var columnRef = GetColumnReference(jsonPath, virtualColumns); - - switch (methodCall.Method.Name) - { - case "Contains": - parameters[paramName] = $"%{value}%"; - return $"{columnRef} LIKE @{paramName}"; - - case "StartsWith": - parameters[paramName] = $"{value}%"; - return $"{columnRef} LIKE @{paramName}"; - - case "EndsWith": - parameters[paramName] = $"%{value}"; - return $"{columnRef} LIKE @{paramName}"; - - default: - throw new NotSupportedException($"String method '{methodCall.Method.Name}' is not supported"); - } - } - - throw new NotSupportedException($"Method '{methodCall.Method.Name}' is not supported"); - } - - /// - /// Gets the SQL column reference for a JSON path. - /// Returns the virtual column name if one exists, otherwise returns json_extract(). - /// - private static string GetColumnReference( - string jsonPath, - IReadOnlyDictionary? virtualColumns) - { - if (virtualColumns != null && virtualColumns.TryGetValue(jsonPath, out var columnInfo)) - { - return $"[{columnInfo.ColumnName}]"; - } - - return $"json_extract(data, '{jsonPath}')"; - } - - private static (string jsonPath, object value) ExtractComparisonParts(BinaryExpression binary) - { - // Determine which side is the member access and which is the constant - Expression memberExpr; - Expression valueExpr; - - if (IsMemberOrPropertyAccess(binary.Left)) - { - memberExpr = binary.Left; - valueExpr = binary.Right; - } - else if (IsMemberOrPropertyAccess(binary.Right)) - { - memberExpr = binary.Right; - valueExpr = binary.Left; - } - else - { - throw new NotSupportedException("At least one side of the comparison must be a property access"); - } - - var jsonPath = TranslateToJsonPath(memberExpr); - var value = GetConstantValue(valueExpr); - - return (jsonPath, value); - } - - private static string TranslateToJsonPath(Expression expression) - { - // Handle boxing conversions - if (expression is UnaryExpression { NodeType: ExpressionType.Convert } unary) - { - expression = unary.Operand; - } - - var path = new StringBuilder("$"); - BuildPath(expression, path); - return path.ToString(); - } - - private static bool IsMemberOrPropertyAccess(Expression expression) - { - if (expression is UnaryExpression { NodeType: ExpressionType.Convert } unary) - { - expression = unary.Operand; - } - - return expression is MemberExpression; - } - - private static object GetConstantValue(Expression expression) - { - // Handle boxing conversions - if (expression is UnaryExpression { NodeType: ExpressionType.Convert } unary) - { - expression = unary.Operand; - } - - // Handle constant expressions - if (expression is ConstantExpression constant) - { - return constant.Value ?? throw new InvalidOperationException("Constant value cannot be null"); - } - - // Handle member access chains using reflection (avoids expensive Expression.Compile()) - // This handles patterns like: closure.field, closure.field.property, obj.Property.SubProperty - if (expression is MemberExpression memberExpr) - { - return EvaluateMemberExpression(memberExpr); - } - - // Handle method calls (e.g., ToString(), GetValue()) - if (expression is MethodCallExpression methodCall) - { - return EvaluateMethodCall(methodCall); - } - - // Fallback: Compile and execute the expression to get the value - var lambda = Expression.Lambda>(Expression.Convert(expression, typeof(object))); - var compiled = lambda.Compile(); - return compiled() ?? throw new InvalidOperationException("Expression evaluated to null"); - } - - /// - /// Evaluates a member expression chain using reflection. - /// Handles nested property/field access like: closure.field.Property.SubProperty - /// - private static object EvaluateMemberExpression(MemberExpression memberExpr) - { - // Build the chain of member accesses from innermost to outermost - var memberChain = new List(); - Expression? current = memberExpr; - - while (current is MemberExpression member) - { - memberChain.Add(member.Member); - current = member.Expression; - } - - // The innermost expression should be a constant (the closure or static field) - object? value; - if (current is ConstantExpression constantExpr) - { - value = constantExpr.Value; - } - else if (current is null) - { - // Static member access - start with null - value = null; - } - else - { - throw new NotSupportedException( - $"Member expression must be rooted in a constant. Got: {current.NodeType}"); - } - - // Traverse the chain from innermost to outermost (reverse order) - for (int i = memberChain.Count - 1; i >= 0; i--) - { - var member = memberChain[i]; - value = member switch - { - FieldInfo field => field.GetValue(value), - PropertyInfo property => property.GetValue(value), - _ => throw new NotSupportedException($"Unsupported member type: {member.MemberType}") - }; - } - - return value ?? throw new InvalidOperationException("Expression evaluated to null"); - } - - /// - /// Evaluates simple method calls using reflection. - /// Supports instance methods on evaluated objects and static methods. - /// - private static object EvaluateMethodCall(MethodCallExpression methodCall) - { - // Evaluate the instance (if not static) - object? instance = null; - if (methodCall.Object != null) - { - instance = GetConstantValue(methodCall.Object); - } - - // Evaluate arguments - var args = new object?[methodCall.Arguments.Count]; - for (int i = 0; i < methodCall.Arguments.Count; i++) - { - args[i] = GetConstantValue(methodCall.Arguments[i]); - } - - // Invoke the method - var result = methodCall.Method.Invoke(instance, args); - return result ?? throw new InvalidOperationException($"Method '{methodCall.Method.Name}' returned null"); - } - - private static bool IsIndexerAccess(MethodCallExpression methodCall) - { - return methodCall.Method.Name == "get_Item" && - methodCall.Object != null && - methodCall.Arguments.Count == 1; - } - - private static int GetIndexValue(Expression expression) - { - var value = GetConstantValue(expression); - return value is int index ? index : throw new NotSupportedException("Array index must be an integer"); - } - - private static bool IsComparisonOperator(ExpressionType nodeType) - { - return nodeType is ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual - or ExpressionType.LessThan or ExpressionType.LessThanOrEqual; - } - - private static string ToCamelCase(string propertyName) - { - if (string.IsNullOrEmpty(propertyName)) - return propertyName; - - // Convert PascalCase to camelCase (FirstName -> firstName) - return char.ToLowerInvariant(propertyName[0]) + propertyName.Substring(1); - } - - /// - /// Extracts field selections from a projection expression. - /// Supports object initializer syntax: x => new DTO { Name = x.Name, Email = x.Email } - /// - /// The source type - /// The result type - /// The projection expression - /// Dictionary mapping result property names to JSON paths - public static Dictionary ExtractFieldSelections( - Expression> selector) - { - ArgumentNullException.ThrowIfNull(selector); - - var body = selector.Body; - - // Handle boxing conversions - if (body is UnaryExpression { NodeType: ExpressionType.Convert } unary) - { - body = unary.Operand; - } - - // Support MemberInitExpression: new DTO { Name = x.Name, Email = x.Email } - if (body is MemberInitExpression memberInit) - { - return ExtractFromMemberInit(memberInit); - } - - // Support NewExpression: new { x.Name, x.Email } (anonymous types) - if (body is NewExpression newExpr) - { - return ExtractFromNewExpression(newExpr); - } - - throw new NotSupportedException( - "Only member initialization (new DTO { Name = x.Name }) and anonymous type creation (new { x.Name }) are supported for projection"); - } - - private static Dictionary ExtractFromMemberInit(MemberInitExpression memberInit) - { - var selections = new Dictionary(); - - foreach (var binding in memberInit.Bindings) - { - if (binding is not MemberAssignment assignment) - { - throw new NotSupportedException($"Only member assignments are supported in projections"); - } - - var propertyName = assignment.Member.Name; - var jsonPath = TranslateToJsonPath(assignment.Expression); - selections[propertyName] = jsonPath; - } - - return selections; - } - - private static Dictionary ExtractFromNewExpression(NewExpression newExpr) - { - var selections = new Dictionary(); - - // Anonymous types have both Arguments and Members - if (newExpr.Members == null || newExpr.Members.Count != newExpr.Arguments.Count) - { - throw new NotSupportedException("Constructor expression must have matching members and arguments"); - } - - for (int i = 0; i < newExpr.Members.Count; i++) - { - var propertyName = newExpr.Members[i].Name; - var jsonPath = TranslateToJsonPath(newExpr.Arguments[i]); - selections[propertyName] = jsonPath; - } - - return selections; - } -} diff --git a/src/LiteDocumentStore/Core/IDocumentStore.cs b/src/LiteDocumentStore/Core/IDocumentStore.cs index eeb8075..735fe1a 100644 --- a/src/LiteDocumentStore/Core/IDocumentStore.cs +++ b/src/LiteDocumentStore/Core/IDocumentStore.cs @@ -163,55 +163,6 @@ Task AddVirtualColumnAsync( /// An enumerable of deserialized objects matching the query Task> QueryAsync(string jsonPath, TValue value); - /// - /// Queries documents using a LINQ expression predicate that gets translated to SQLite json_extract() queries. - /// Supports property access, nested properties, comparisons, and logical operators. - /// - /// Type of the objects to retrieve (also used as table name) - /// The predicate expression to filter documents - /// An enumerable of deserialized objects matching the query - Task> QueryAsync(System.Linq.Expressions.Expression> predicate); - - /// - /// Selects specific JSON fields from documents and projects them into a result type. - /// This is more efficient than retrieving entire documents when only certain fields are needed. - /// Uses json_extract() to retrieve only the specified fields from the JSONB data. - /// - /// Type of the source documents (also used as table name) - /// Type to project the selected fields into - /// Expression selecting fields to project (e.g., x => new { x.Name, x.Email }) - /// An enumerable of projected objects with only the selected fields - /// - /// - /// // Select only name and email fields from Customer documents - /// var results = await store.SelectAsync<Customer, CustomerDto>( - /// x => new CustomerDto { Name = x.Name, Email = x.Email }); - /// - /// - Task> SelectAsync(System.Linq.Expressions.Expression> selector); - - /// - /// Selects specific JSON fields from documents that match a predicate and projects them into a result type. - /// Combines filtering and projection for optimal performance - only matching documents are retrieved and - /// only specified fields are extracted. - /// - /// Type of the source documents (also used as table name) - /// Type to project the selected fields into - /// Expression to filter documents - /// Expression selecting fields to project - /// An enumerable of projected objects with only the selected fields - /// - /// - /// // Select name and email from active customers only - /// var results = await store.SelectAsync<Customer, CustomerDto>( - /// x => x.Active == true, - /// x => new CustomerDto { Name = x.Name, Email = x.Email }); - /// - /// - Task> SelectAsync( - System.Linq.Expressions.Expression> predicate, - System.Linq.Expressions.Expression> selector); - /// /// Gets the underlying SQLite connection for advanced operations and raw SQL access. /// This enables the hybrid experience where users can use both document storage diff --git a/src/LiteDocumentStore/Core/SqliteCommandExtensions.cs b/src/LiteDocumentStore/Core/SqliteCommandExtensions.cs new file mode 100644 index 0000000..5e106f1 --- /dev/null +++ b/src/LiteDocumentStore/Core/SqliteCommandExtensions.cs @@ -0,0 +1,135 @@ +using System.Data; +using System.Globalization; +using Microsoft.Data.Sqlite; + +namespace LiteDocumentStore; + +/// +/// Internal, reflection-free ADO.NET helpers over . +/// These replace the previous Dapper dependency: parameters are bound explicitly and +/// results are read by ordinal, so nothing here relies on runtime reflection or IL +/// generation (AOT/trim safe). +/// +/// +/// Commands are created with , which assigns +/// the connection's currently active transaction automatically, so callers do not need to +/// pass a transaction explicitly to participate in one. +/// +internal static class SqliteCommandExtensions +{ + /// + /// Executes a non-query statement and returns the number of affected rows. + /// + public static async Task ExecuteAsync( + this SqliteConnection connection, + string commandText, + params (string Name, object? Value)[] parameters) + { + await using var command = CreateCommand(connection, commandText, parameters); + return await command.ExecuteNonQueryAsync().ConfigureAwait(false); + } + + /// + /// Executes a non-query statement synchronously and returns the number of affected rows. + /// + public static int Execute( + this SqliteConnection connection, + string commandText, + params (string Name, object? Value)[] parameters) + { + using var command = CreateCommand(connection, commandText, parameters); + return command.ExecuteNonQuery(); + } + + /// + /// Executes a query and returns the first column of the first row converted to + /// , or default when there is no row or the value is NULL. + /// + public static async Task ExecuteScalarAsync( + this SqliteConnection connection, + string commandText, + params (string Name, object? Value)[] parameters) + { + await using var command = CreateCommand(connection, commandText, parameters); + var result = await command.ExecuteScalarAsync().ConfigureAwait(false); + return ConvertScalar(result); + } + + /// + /// Executes a query and returns the first column of every row as strings + /// (NULL values are preserved as null). Used for reading json(data) documents. + /// + public static async Task> QueryStringsAsync( + this SqliteConnection connection, + string commandText, + params (string Name, object? Value)[] parameters) + { + await using var command = CreateCommand(connection, commandText, parameters); + await using var reader = await command.ExecuteReaderAsync().ConfigureAwait(false); + + var results = new List(); + while (await reader.ReadAsync().ConfigureAwait(false)) + { + results.Add(reader.IsDBNull(0) ? null : reader.GetString(0)); + } + + return results; + } + + /// + /// Executes a query and returns the first column of the first row as a string, + /// or null when there is no row or the value is NULL. + /// + public static async Task QueryFirstStringAsync( + this SqliteConnection connection, + string commandText, + params (string Name, object? Value)[] parameters) + { + await using var command = CreateCommand(connection, commandText, parameters); + var result = await command.ExecuteScalarAsync().ConfigureAwait(false); + return result is null or DBNull ? null : Convert.ToString(result, CultureInfo.InvariantCulture); + } + + /// + /// Synchronous variant of , used on the disposal path. + /// + public static string? QueryFirstString(this SqliteConnection connection, string commandText) + { + using var command = CreateCommand(connection, commandText, []); + var result = command.ExecuteScalar(); + return result is null or DBNull ? null : Convert.ToString(result, CultureInfo.InvariantCulture); + } + + private static SqliteCommand CreateCommand( + SqliteConnection connection, + string commandText, + (string Name, object? Value)[] parameters) + { + var command = connection.CreateCommand(); + command.CommandText = commandText; + + foreach (var (name, value) in parameters) + { + var parameterName = name.StartsWith('@') ? name : "@" + name; + command.Parameters.AddWithValue(parameterName, value ?? DBNull.Value); + } + + return command; + } + + private static T? ConvertScalar(object? result) + { + if (result is null or DBNull) + { + return default; + } + + var targetType = Nullable.GetUnderlyingType(typeof(T)) ?? typeof(T); + if (result.GetType() == targetType) + { + return (T)result; + } + + return (T)Convert.ChangeType(result, targetType, CultureInfo.InvariantCulture); + } +} diff --git a/src/LiteDocumentStore/Core/VirtualColumnCache.cs b/src/LiteDocumentStore/Core/VirtualColumnCache.cs deleted file mode 100644 index 2794e0f..0000000 --- a/src/LiteDocumentStore/Core/VirtualColumnCache.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System.Collections.Concurrent; -using System.Text.RegularExpressions; -using Microsoft.Data.Sqlite; - -namespace LiteDocumentStore; - -/// -/// Contains information about a virtual column mapped to a JSON path. -/// -/// The JSON path expression (e.g., '$.Category') -/// The virtual column name in the table -/// The SQLite column type (e.g., TEXT, INTEGER, REAL) -public sealed record VirtualColumnInfo(string JsonPath, string ColumnName, string ColumnType); - -/// -/// Caches virtual column information per table to enable query optimization. -/// Automatically discovers existing virtual columns from the database schema on first access -/// and updates the cache when new virtual columns are added. -/// -internal sealed partial class VirtualColumnCache -{ - private readonly SqliteConnection _connection; - private readonly ConcurrentDictionary> _cache = new(); - - // Regex to extract json_extract path from generated column definition - // Matches: json_extract(data, '$.Path.To.Property') - [GeneratedRegex(@"json_extract\s*\(\s*data\s*,\s*'(\$\.[^']+)'\s*\)", RegexOptions.IgnoreCase | RegexOptions.Compiled)] - private static partial Regex JsonExtractPattern(); - - /// - /// Initializes a new virtual column cache with the specified connection. - /// - /// The SQLite connection to use for schema introspection - public VirtualColumnCache(SqliteConnection connection) - { - _connection = connection ?? throw new ArgumentNullException(nameof(connection)); - } - - /// - /// Gets virtual columns for a table. Loads from schema on first access. - /// - /// The table name to get virtual columns for - /// A dictionary mapping JSON paths to virtual column info - public async ValueTask> GetAsync(string tableName) - { - ArgumentNullException.ThrowIfNull(tableName); - - if (_cache.TryGetValue(tableName, out var columns)) - { - return columns; - } - - var loaded = await LoadFromSchemaAsync(tableName).ConfigureAwait(false); - _cache[tableName] = loaded; - return loaded; - } - - /// - /// Registers a newly created virtual column, updating the cache without re-querying schema. - /// - /// The table name - /// The virtual column info to register - public void Register(string tableName, VirtualColumnInfo column) - { - ArgumentNullException.ThrowIfNull(tableName); - ArgumentNullException.ThrowIfNull(column); - - _cache.AddOrUpdate( - tableName, - _ => new Dictionary { [column.JsonPath] = column }, - (_, existing) => - { - var updated = new Dictionary(existing) - { - [column.JsonPath] = column - }; - return updated; - }); - } - - /// - /// Invalidates the cache for a specific table, forcing reload on next access. - /// - /// The table name to invalidate - public void Invalidate(string tableName) - { - ArgumentNullException.ThrowIfNull(tableName); - _cache.TryRemove(tableName, out _); - } - - /// - /// Clears the entire cache. - /// - public void Clear() => _cache.Clear(); - - /// - /// Loads virtual column information from the database schema. - /// - private async Task> LoadFromSchemaAsync(string tableName) - { - var result = new Dictionary(); - var introspector = new SchemaIntrospector(_connection); - - // Get all columns for the table - var columns = await introspector.GetColumnsAsync(tableName).ConfigureAwait(false); - var hiddenColumns = columns.Where(c => c.IsHidden).ToList(); - - if (hiddenColumns.Count == 0) - { - return result; - } - - // Get the table SQL to parse generated column expressions - var tables = await introspector.GetTablesAsync().ConfigureAwait(false); - var tableInfo = tables.FirstOrDefault(t => - string.Equals(t.Name, tableName, StringComparison.OrdinalIgnoreCase)); - - if (tableInfo?.Sql == null) - { - return result; - } - - // Parse each hidden column's generation expression from the table SQL - foreach (var column in hiddenColumns) - { - var jsonPath = ExtractJsonPathForColumn(tableInfo.Sql, column.Name); - if (jsonPath != null) - { - result[jsonPath] = new VirtualColumnInfo(jsonPath, column.Name, column.Type); - } - } - - return result; - } - - /// - /// Extracts the JSON path from a column's generated expression in the CREATE TABLE SQL. - /// - private static string? ExtractJsonPathForColumn(string tableSql, string columnName) - { - // Pattern to match: [columnName] TYPE GENERATED ALWAYS AS (json_extract(data, '$.Path')) VIRTUAL - // We need to find the json_extract for this specific column - var columnPattern = $@"\[?{Regex.Escape(columnName)}\]?\s+\w+\s+GENERATED\s+ALWAYS\s+AS\s*\(([^)]+)\)"; - var columnMatch = Regex.Match(tableSql, columnPattern, RegexOptions.IgnoreCase); - - if (!columnMatch.Success) - { - return null; - } - - var expression = columnMatch.Groups[1].Value; - var jsonMatch = JsonExtractPattern().Match(expression); - - return jsonMatch.Success ? jsonMatch.Groups[1].Value : null; - } -} diff --git a/src/LiteDocumentStore/Factories/DefaultConnectionFactory.cs b/src/LiteDocumentStore/Factories/DefaultConnectionFactory.cs index 789222d..ce5961d 100644 --- a/src/LiteDocumentStore/Factories/DefaultConnectionFactory.cs +++ b/src/LiteDocumentStore/Factories/DefaultConnectionFactory.cs @@ -136,23 +136,3 @@ private static string GetSynchronousModeString(SynchronousMode mode) }; } } - -/// -/// Extension methods for SqliteConnection to execute commands. -/// -internal static class SqliteConnectionExtensions -{ - public static void Execute(this SqliteConnection connection, string commandText) - { - using var command = connection.CreateCommand(); - command.CommandText = commandText; - command.ExecuteNonQuery(); - } - - public static async Task ExecuteAsync(this SqliteConnection connection, string commandText) - { - await using var command = connection.CreateCommand(); - command.CommandText = commandText; - await command.ExecuteNonQueryAsync().ConfigureAwait(false); - } -} diff --git a/src/LiteDocumentStore/Factories/DocumentStoreFactory.cs b/src/LiteDocumentStore/Factories/DocumentStoreFactory.cs index 18c1511..f5aa275 100644 --- a/src/LiteDocumentStore/Factories/DocumentStoreFactory.cs +++ b/src/LiteDocumentStore/Factories/DocumentStoreFactory.cs @@ -57,7 +57,7 @@ public IDocumentStore Create(DocumentStoreOptions options) var connection = _connectionFactory.CreateConnection(options); - return new DocumentStore(connection, namingConvention, logger, ownsConnection: true); + return new DocumentStore(connection, namingConvention, logger, ownsConnection: true, options.SerializerOptions); } /// @@ -71,6 +71,6 @@ public async Task CreateAsync(DocumentStoreOptions options) var connection = await _connectionFactory.CreateConnectionAsync(options).ConfigureAwait(false); - return new DocumentStore(connection, namingConvention, logger, ownsConnection: true); + return new DocumentStore(connection, namingConvention, logger, ownsConnection: true, options.SerializerOptions); } } diff --git a/src/LiteDocumentStore/LiteDocumentStore.csproj b/src/LiteDocumentStore/LiteDocumentStore.csproj index df3cf82..f0d6ab7 100644 --- a/src/LiteDocumentStore/LiteDocumentStore.csproj +++ b/src/LiteDocumentStore/LiteDocumentStore.csproj @@ -6,15 +6,16 @@ enable latest true - + true + LiteDocumentStore LiteDocumentStore idotta - A high-performance, single-file application data format using C#, SQLite, and Dapper. Stores JSON data in SQLite's JSONB format for optimal performance. - sqlite;jsonb;json;database;repository;async;dapper;orm;document-store - https://github.com/idotta/jsonb-store - https://github.com/idotta/jsonb-store + A high-performance, single-file application data format using C# and SQLite. Stores JSON data in SQLite's JSONB format for optimal performance. Native AOT / trimming compatible. + sqlite;jsonb;json;database;repository;async;document-store;aot + https://github.com/idotta/lite-doc-store + https://github.com/idotta/lite-doc-store git MIT README.md @@ -27,11 +28,16 @@ - - - - - + + + + + + diff --git a/src/LiteDocumentStore/Migrations/MigrationRunner.cs b/src/LiteDocumentStore/Migrations/MigrationRunner.cs index 88558c2..710850f 100644 --- a/src/LiteDocumentStore/Migrations/MigrationRunner.cs +++ b/src/LiteDocumentStore/Migrations/MigrationRunner.cs @@ -1,5 +1,5 @@ -using System.Data; -using Dapper; +using System.Globalization; +using LiteDocumentStore.Exceptions; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -25,9 +25,6 @@ public MigrationRunner(SqliteConnection connection, ILogger? lo { _connection = connection ?? throw new ArgumentNullException(nameof(connection)); _logger = logger ?? NullLogger.Instance; - - // Register DateTimeOffset handler for Dapper if not already registered - SqlMapper.AddTypeHandler(new DateTimeOffsetHandler()); } /// @@ -55,12 +52,30 @@ public async Task> GetAppliedMigrationsAsync await EnsureMigrationTableExistsAsync().ConfigureAwait(false); var sql = $@" - SELECT version as Version, name as Name, applied_at as AppliedAt - FROM [{MigrationTableName}] + SELECT version, name, applied_at + FROM [{MigrationTableName}] ORDER BY version"; _logger.LogDebug("Retrieving applied migrations"); - var records = await _connection.QueryAsync(sql).ConfigureAwait(false); + + await using var command = _connection.CreateCommand(); + command.CommandText = sql; + await using var reader = await command.ExecuteReaderAsync().ConfigureAwait(false); + + var records = new List(); + while (await reader.ReadAsync().ConfigureAwait(false)) + { + records.Add(new MigrationHistoryRecord + { + Version = reader.GetInt64(0), + Name = reader.GetString(1), + AppliedAt = DateTimeOffset.Parse( + reader.GetString(2), + CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind) + }); + } + return records; } @@ -107,17 +122,18 @@ public async Task ApplyMigrationAsync(IMigration migration) // Execute the migration await migration.UpAsync(_connection).ConfigureAwait(false); - // Record the migration + // Record the migration. Commands created on the connection automatically enlist + // in the active transaction. Persist DateTimeOffset as an ISO-8601 round-trip string. var sql = $@" - INSERT INTO [{MigrationTableName}] (version, name, applied_at) + INSERT INTO [{MigrationTableName}] (version, name, applied_at) VALUES (@Version, @Name, @AppliedAt)"; - await _connection.ExecuteAsync(sql, new - { - Version = migration.Version, - Name = migration.Name, - AppliedAt = DateTimeOffset.UtcNow - }, transaction).ConfigureAwait(false); + await _connection.ExecuteAsync( + sql, + ("Version", migration.Version), + ("Name", migration.Name), + ("AppliedAt", DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture))) + .ConfigureAwait(false); transaction.Commit(); _logger.LogInformation("Migration {Version} applied successfully", migration.Version); @@ -170,7 +186,7 @@ public async Task RollbackMigrationAsync(IMigration migration) // Check if this migration is actually applied var checkSql = $@"SELECT COUNT(*) FROM [{MigrationTableName}] WHERE version = @Version"; - var isApplied = await _connection.ExecuteScalarAsync(checkSql, new { Version = migration.Version }) + var isApplied = await _connection.ExecuteScalarAsync(checkSql, ("Version", migration.Version)) .ConfigureAwait(false) > 0; if (!isApplied) @@ -188,9 +204,9 @@ public async Task RollbackMigrationAsync(IMigration migration) // Execute the rollback await migration.DownAsync(_connection).ConfigureAwait(false); - // Remove the migration record + // Remove the migration record (enlists in the active transaction automatically) var deleteSql = $@"DELETE FROM [{MigrationTableName}] WHERE version = @Version"; - await _connection.ExecuteAsync(deleteSql, new { Version = migration.Version }, transaction) + await _connection.ExecuteAsync(deleteSql, ("Version", migration.Version)) .ConfigureAwait(false); transaction.Commit(); @@ -229,17 +245,29 @@ public async Task RollbackToVersionAsync(long targetVersion, IEnumerable m.Version); - var rolledBackCount = 0; - foreach (var record in migrationsToRollback) + // Fail before mutating anything: every migration in the rollback range must have a + // definition. Rolling back only part of the range would leave the schema and the history + // table in an inconsistent state, so refuse the whole operation instead of skipping. + var missingVersions = migrationsToRollback + .Where(record => !migrationDict.ContainsKey(record.Version)) + .Select(record => record.Version) + .ToList(); + + if (missingVersions.Count > 0) { - if (!migrationDict.TryGetValue(record.Version, out var migration)) - { - _logger.LogWarning("Migration {Version} is applied but definition not found, skipping rollback", - record.Version); - continue; - } + var versions = string.Join(", ", missingVersions); + _logger.LogError( + "Cannot roll back to version {Target}: missing migration definition(s) for version(s) {Versions}", + targetVersion, versions); + throw new LiteDocumentStoreException( + $"Cannot roll back to version {targetVersion}: missing migration definition(s) for version(s) {versions}. No migrations were rolled back."); + } + var rolledBackCount = 0; + foreach (var record in migrationsToRollback) + { + var migration = migrationDict[record.Version]; var rolledBack = await RollbackMigrationAsync(migration).ConfigureAwait(false); if (rolledBack) { diff --git a/src/LiteDocumentStore/Migrations/SchemaIntrospector.cs b/src/LiteDocumentStore/Migrations/SchemaIntrospector.cs index ed12196..890d1d7 100644 --- a/src/LiteDocumentStore/Migrations/SchemaIntrospector.cs +++ b/src/LiteDocumentStore/Migrations/SchemaIntrospector.cs @@ -1,4 +1,3 @@ -using Dapper; using Microsoft.Data.Sqlite; namespace LiteDocumentStore; @@ -26,14 +25,29 @@ public SchemaIntrospector(SqliteConnection connection) /// An enumerable of table information records public async Task> GetTablesAsync() { - var sql = @" - SELECT name as Name, type as Type, sql as Sql - FROM sqlite_master - WHERE type = 'table' + const string sql = @" + SELECT name, type, sql + FROM sqlite_master + WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name"; - return await _connection.QueryAsync(sql).ConfigureAwait(false); + await using var command = _connection.CreateCommand(); + command.CommandText = sql; + await using var reader = await command.ExecuteReaderAsync().ConfigureAwait(false); + + var tables = new List(); + while (await reader.ReadAsync().ConfigureAwait(false)) + { + tables.Add(new TableInfo + { + Name = reader.GetString(0), + Type = reader.GetString(1), + Sql = reader.IsDBNull(2) ? null : reader.GetString(2) + }); + } + + return tables; } /// @@ -45,13 +59,13 @@ public async Task TableExistsAsync(string tableName) { ArgumentNullException.ThrowIfNull(tableName); - var sql = @" - SELECT COUNT(*) - FROM sqlite_master - WHERE type = 'table' + const string sql = @" + SELECT COUNT(*) + FROM sqlite_master + WHERE type = 'table' AND name = @TableName"; - var count = await _connection.ExecuteScalarAsync(sql, new { TableName = tableName }) + var count = await _connection.ExecuteScalarAsync(sql, ("TableName", tableName)) .ConfigureAwait(false); return count > 0; @@ -67,20 +81,42 @@ public async Task> GetColumnsAsync(string tableName) { ArgumentNullException.ThrowIfNull(tableName); - // Use table_xinfo instead of table_info to include generated columns - var sql = $"PRAGMA table_xinfo([{tableName}])"; - var pragmaResults = await _connection.QueryAsync(sql).ConfigureAwait(false); - - return pragmaResults.Select(row => new ColumnInfo + // Use table_xinfo instead of table_info to include generated columns. + // PRAGMA arguments cannot be parameterized, so the identifier is interpolated. SQLite's + // [ ] quoting has no escape for a literal ']', so use standard double-quote identifier + // quoting and escape any embedded quote ("") to prevent breaking out of the identifier. + var quotedTable = "\"" + tableName.Replace("\"", "\"\"") + "\""; + var sql = $"PRAGMA table_xinfo({quotedTable})"; + + await using var command = _connection.CreateCommand(); + command.CommandText = sql; + await using var reader = await command.ExecuteReaderAsync().ConfigureAwait(false); + + // Resolve column ordinals by name to be resilient to PRAGMA column ordering. + var cidOrdinal = reader.GetOrdinal("cid"); + var nameOrdinal = reader.GetOrdinal("name"); + var typeOrdinal = reader.GetOrdinal("type"); + var notNullOrdinal = reader.GetOrdinal("notnull"); + var defaultOrdinal = reader.GetOrdinal("dflt_value"); + var pkOrdinal = reader.GetOrdinal("pk"); + var hiddenOrdinal = reader.GetOrdinal("hidden"); + + var columns = new List(); + while (await reader.ReadAsync().ConfigureAwait(false)) { - ColumnId = (long)row.cid, - Name = (string)row.name, - Type = (string)row.type, - NotNull = (long)row.notnull == 1, - DefaultValue = row.dflt_value, - IsPrimaryKey = (long)row.pk == 1, - IsHidden = (long)row.hidden != 0 // hidden=1 for virtual, hidden=2 for stored - }); + columns.Add(new ColumnInfo + { + ColumnId = reader.GetInt64(cidOrdinal), + Name = reader.GetString(nameOrdinal), + Type = reader.GetString(typeOrdinal), + NotNull = reader.GetInt64(notNullOrdinal) == 1, + DefaultValue = reader.IsDBNull(defaultOrdinal) ? null : reader.GetValue(defaultOrdinal), + IsPrimaryKey = reader.GetInt64(pkOrdinal) == 1, + IsHidden = reader.GetInt64(hiddenOrdinal) != 0 // hidden=1 for virtual, hidden=2 for stored + }); + } + + return columns; } /// @@ -91,20 +127,40 @@ public async Task> GetColumnsAsync(string tableName) public async Task> GetIndexesAsync(string? tableName = null) { var sql = @" - SELECT name as Name, tbl_name as TableName, sql as Sql - FROM sqlite_master - WHERE type = 'index' + SELECT name, tbl_name, sql + FROM sqlite_master + WHERE type = 'index' AND name NOT LIKE 'sqlite_%'"; - if (!string.IsNullOrEmpty(tableName)) + var filterByTable = !string.IsNullOrEmpty(tableName); + if (filterByTable) { sql += " AND tbl_name = @TableName"; } sql += " ORDER BY name"; - return await _connection.QueryAsync(sql, new { TableName = tableName }) - .ConfigureAwait(false); + await using var command = _connection.CreateCommand(); + command.CommandText = sql; + if (filterByTable) + { + command.Parameters.AddWithValue("@TableName", tableName); + } + + await using var reader = await command.ExecuteReaderAsync().ConfigureAwait(false); + + var indexes = new List(); + while (await reader.ReadAsync().ConfigureAwait(false)) + { + indexes.Add(new IndexInfo + { + Name = reader.GetString(0), + TableName = reader.GetString(1), + Sql = reader.IsDBNull(2) ? null : reader.GetString(2) + }); + } + + return indexes; } /// @@ -116,13 +172,13 @@ public async Task IndexExistsAsync(string indexName) { ArgumentNullException.ThrowIfNull(indexName); - var sql = @" - SELECT COUNT(*) - FROM sqlite_master - WHERE type = 'index' + const string sql = @" + SELECT COUNT(*) + FROM sqlite_master + WHERE type = 'index' AND name = @IndexName"; - var count = await _connection.ExecuteScalarAsync(sql, new { IndexName = indexName }) + var count = await _connection.ExecuteScalarAsync(sql, ("IndexName", indexName)) .ConfigureAwait(false); return count > 0; @@ -149,8 +205,7 @@ public async Task ColumnExistsAsync(string tableName, string columnName) /// The SQLite version string public async Task GetSqliteVersionAsync() { - var sql = "SELECT sqlite_version()"; - var version = await _connection.ExecuteScalarAsync(sql).ConfigureAwait(false); + var version = await _connection.QueryFirstStringAsync("SELECT sqlite_version()").ConfigureAwait(false); return version ?? "Unknown"; } @@ -160,13 +215,9 @@ public async Task GetSqliteVersionAsync() /// Database statistics public async Task GetDatabaseStatisticsAsync() { - var pageCountSql = "PRAGMA page_count"; - var pageSizeSql = "PRAGMA page_size"; - var freeSql = "PRAGMA freelist_count"; - - var pageCount = await _connection.ExecuteScalarAsync(pageCountSql).ConfigureAwait(false); - var pageSize = await _connection.ExecuteScalarAsync(pageSizeSql).ConfigureAwait(false); - var freePages = await _connection.ExecuteScalarAsync(freeSql).ConfigureAwait(false); + var pageCount = await _connection.ExecuteScalarAsync("PRAGMA page_count").ConfigureAwait(false); + var pageSize = await _connection.ExecuteScalarAsync("PRAGMA page_size").ConfigureAwait(false); + var freePages = await _connection.ExecuteScalarAsync("PRAGMA freelist_count").ConfigureAwait(false); return new DatabaseStatistics { diff --git a/src/LiteDocumentStore/Serialization/JsonHelper.cs b/src/LiteDocumentStore/Serialization/JsonHelper.cs index 5c5268d..3e46695 100644 --- a/src/LiteDocumentStore/Serialization/JsonHelper.cs +++ b/src/LiteDocumentStore/Serialization/JsonHelper.cs @@ -1,41 +1,52 @@ +using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; using LiteDocumentStore.Exceptions; namespace LiteDocumentStore; /// -/// Internal helper for high-performance JSON serialization optimized for SQLite JSONB storage. -/// Uses fixed, optimized JsonSerializerOptions for maximum performance. +/// Internal helper for JSON serialization optimized for SQLite JSONB storage. +/// All (de)serialization goes through the AOT-safe overloads, +/// resolving the type metadata from the caller-provided . +/// AOT consumers supply a source-generated via +/// ; when none is supplied the store +/// falls back to (non-AOT only). /// internal static class JsonHelper { /// - /// Optimized JSON serializer options for performance. - /// - PropertyNameCaseInsensitive = false (exact match for speed) - /// - DefaultIgnoreCondition = WhenWritingNull (smaller JSON) - /// - WriteIndented = false (compact output) + /// Builds the reflection-based fallback options used when the consumer does not supply + /// their own . This is the single quarantined spot for + /// reflection-based serialization: it is not AOT/trim safe and is only reached on the + /// fallback path. AOT consumers always provide a source-generated context and never hit it. /// - private static readonly JsonSerializerOptions Options = new() + [UnconditionalSuppressMessage("Trimming", "IL2026", + Justification = "Reflection-based JSON is the documented non-AOT fallback; AOT consumers supply a source-generated JsonSerializerContext via DocumentStoreOptions.SerializerOptions.")] + [UnconditionalSuppressMessage("AOT", "IL3050", + Justification = "Reflection-based JSON is the documented non-AOT fallback; AOT consumers supply a source-generated JsonSerializerContext via DocumentStoreOptions.SerializerOptions.")] + public static JsonSerializerOptions CreateDefaultReflectionOptions() { - PropertyNameCaseInsensitive = false, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - WriteIndented = false - }; + return new JsonSerializerOptions + { + PropertyNameCaseInsensitive = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + TypeInfoResolver = new DefaultJsonTypeInfoResolver() + }; + } /// /// Serializes an object to UTF-8 encoded JSON bytes for JSONB storage. - /// This is the most efficient path for SQLite JSONB operations. /// - /// The type of object to serialize - /// The object to serialize - /// UTF-8 encoded JSON bytes /// Thrown when serialization fails - public static byte[] SerializeToUtf8Bytes(T value) + public static byte[] SerializeToUtf8Bytes(T value, JsonSerializerOptions options) { try { - return JsonSerializer.SerializeToUtf8Bytes(value, Options); + var typeInfo = (JsonTypeInfo)options.GetTypeInfo(typeof(T)); + return JsonSerializer.SerializeToUtf8Bytes(value, typeInfo); } catch (JsonException ex) { @@ -56,11 +67,8 @@ public static byte[] SerializeToUtf8Bytes(T value) /// /// Deserializes UTF-8 encoded JSON bytes to a typed object. /// - /// The type of object to deserialize to - /// The UTF-8 encoded JSON bytes - /// The deserialized object, or default if input is empty /// Thrown when deserialization fails - public static T? Deserialize(ReadOnlySpan utf8Json) + public static T? Deserialize(ReadOnlySpan utf8Json, JsonSerializerOptions options) { if (utf8Json.IsEmpty) { @@ -69,7 +77,8 @@ public static byte[] SerializeToUtf8Bytes(T value) try { - return JsonSerializer.Deserialize(utf8Json, Options); + var typeInfo = (JsonTypeInfo)options.GetTypeInfo(typeof(T)); + return JsonSerializer.Deserialize(utf8Json, typeInfo); } catch (JsonException ex) { @@ -84,11 +93,8 @@ public static byte[] SerializeToUtf8Bytes(T value) /// Deserializes a JSON string to a typed object. /// This overload handles string data from the database. /// - /// The type of object to deserialize to - /// The JSON string - /// The deserialized object, or default if input is null or empty /// Thrown when deserialization fails - public static T? Deserialize(string? json) + public static T? Deserialize(string? json, JsonSerializerOptions options) { if (string.IsNullOrEmpty(json)) { @@ -97,7 +103,8 @@ public static byte[] SerializeToUtf8Bytes(T value) try { - return JsonSerializer.Deserialize(json, Options); + var typeInfo = (JsonTypeInfo)options.GetTypeInfo(typeof(T)); + return JsonSerializer.Deserialize(json, typeInfo); } catch (JsonException ex) { diff --git a/src/LiteDocumentStore/TypeHandlers/DateTimeOffsetHandler.cs b/src/LiteDocumentStore/TypeHandlers/DateTimeOffsetHandler.cs deleted file mode 100644 index 50c3a44..0000000 --- a/src/LiteDocumentStore/TypeHandlers/DateTimeOffsetHandler.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System.Data; -using Dapper; -using LiteDocumentStore.Exceptions; - -namespace LiteDocumentStore; - -/// -/// A Dapper TypeHandler that serializes and deserializes DateTimeOffset values -/// to/from ISO 8601 string format for reliable storage in SQLite. -/// -public sealed class DateTimeOffsetHandler : SqlMapper.TypeHandler -{ - /// - /// Parses a DateTimeOffset from its string representation in the database. - /// - /// The value from the database to parse. - /// The parsed DateTimeOffset. - /// In case the value cannot be parsed as a DateTimeOffset. - public override DateTimeOffset Parse(object value) - { - if (value is string strValue) - { - if (DateTimeOffset.TryParse(strValue, out var result)) - { - return result; - } - throw new SerializationException($"Invalid DateTimeOffset value: {strValue}"); - } - - throw new SerializationException($"Unsupported DateTimeOffset value type: {value.GetType().Name}"); - } - - /// - /// Sets a DateTimeOffset value into the database parameter as an ISO 8601 string. - /// - /// The database parameter to set the value on. - /// The DateTimeOffset value to set. - public override void SetValue(IDbDataParameter parameter, DateTimeOffset value) - { - // Store as a TEXT string in ISO8601 format for reliable storage across time zones. - parameter.Value = value.UtcDateTime.ToString("o"); // "o" is the round-trip format specifier (ISO 8601) - parameter.DbType = DbType.String; - } -} \ No newline at end of file diff --git a/src/tests/LiteDocumentStore.Benchmarks/ComparisonBenchmark.cs b/src/tests/LiteDocumentStore.Benchmarks/ComparisonBenchmark.cs index bd26553..c77e260 100644 --- a/src/tests/LiteDocumentStore.Benchmarks/ComparisonBenchmark.cs +++ b/src/tests/LiteDocumentStore.Benchmarks/ComparisonBenchmark.cs @@ -267,7 +267,7 @@ public int LiteDB_FullScan() [Benchmark] public async Task LiteDocumentStore_QueryByCategory() { - var results = await _documentStore.QueryAsync(d => d.Category == "Category 5"); + var results = await _documentStore.QueryAsync("$.Category", "Category 5"); return results.Count(); } diff --git a/src/tests/LiteDocumentStore.Benchmarks/LiteDocumentStore.Benchmarks.csproj b/src/tests/LiteDocumentStore.Benchmarks/LiteDocumentStore.Benchmarks.csproj index 7c20709..3c6e2c5 100644 --- a/src/tests/LiteDocumentStore.Benchmarks/LiteDocumentStore.Benchmarks.csproj +++ b/src/tests/LiteDocumentStore.Benchmarks/LiteDocumentStore.Benchmarks.csproj @@ -10,10 +10,12 @@ - + - - + + + + diff --git a/src/tests/LiteDocumentStore.Benchmarks/ProjectionQueryBenchmark.cs b/src/tests/LiteDocumentStore.Benchmarks/ProjectionQueryBenchmark.cs deleted file mode 100644 index 3e902ef..0000000 --- a/src/tests/LiteDocumentStore.Benchmarks/ProjectionQueryBenchmark.cs +++ /dev/null @@ -1,234 +0,0 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Engines; -using Microsoft.Data.Sqlite; -using Microsoft.Extensions.DependencyInjection; - -namespace LiteDocumentStore.Benchmarks; - -/// -/// Benchmarks comparing full document retrieval vs projection queries. -/// -[MemoryDiagnoser] -[SimpleJob(RunStrategy.Throughput, iterationCount: 15)] -public class ProjectionQueryBenchmark -{ - private IDocumentStore _store = null!; - private ServiceProvider _serviceProvider = null!; - private const int DocumentCount = 1000; - - [GlobalSetup] - public async Task Setup() - { - // Setup DI container with in-memory database - var services = new ServiceCollection(); - services.AddLiteDocumentStore(options => - { - options.ConnectionString = "Data Source=:memory:"; - options.EnableWalMode = false; // WAL not supported in :memory: - }); - - _serviceProvider = services.BuildServiceProvider(); - _store = _serviceProvider.GetRequiredService(); - - // Create table and seed with realistic data - await _store.CreateTableAsync(); - - var documents = new List<(string id, LargeDocument data)>(); - for (int i = 0; i < DocumentCount; i++) - { - var doc = new LargeDocument - { - Id = $"doc-{i}", - Name = $"Document {i}", - Email = $"user{i}@example.com", - Description = $"This is a detailed description for document {i} with lots of text to make it more realistic and increase the payload size significantly.", - Category = $"Category {i % 10}", - Tags = [$"tag{i}", $"tag{i + 1}", $"tag{i + 2}"], - Metadata = new Dictionary - { - ["CreatedBy"] = $"User {i % 100}", - ["Department"] = $"Dept {i % 20}", - ["Version"] = "1.0", - ["Status"] = i % 3 == 0 ? "Active" : "Inactive" - }, - Content = new ContentBlock - { - Title = $"Content Title {i}", - Body = string.Join(" ", Enumerable.Range(0, 50).Select(j => $"Word{j}")), - Author = $"Author {i % 50}", - PublishDate = DateTime.Now.AddDays(-i), - ViewCount = i * 10, - LikeCount = i * 2 - }, - Attachments = Enumerable.Range(0, 5).Select(j => new Attachment - { - FileName = $"file{j}.pdf", - FileSize = 1024 * (j + 1), - MimeType = "application/pdf", - Url = $"https://example.com/files/file{j}.pdf" - }).ToList(), - Timestamps = new TimestampInfo - { - CreatedAt = DateTime.Now.AddDays(-i), - UpdatedAt = DateTime.Now.AddDays(-i / 2), - LastAccessedAt = DateTime.Now - }, - Stats = new Statistics - { - TotalViews = i * 100, - UniqueVisitors = i * 50, - AvgTimeOnPage = TimeSpan.FromSeconds(i % 300), - BounceRate = i % 100 / 100.0 - } - }; - - documents.Add((doc.Id, doc)); - } - - // Bulk insert for faster setup - await _store.UpsertManyAsync(documents); - } - - [GlobalCleanup] - public async Task Cleanup() - { - if (_store != null) - { - await _store.DisposeAsync(); - } - _serviceProvider?.Dispose(); - } - - [Benchmark(Baseline = true, Description = "Full document retrieval (baseline)")] - public async Task GetAllAsync_FullDocuments() - { - var documents = await _store.GetAllAsync(); - return documents.Count(); - } - - [Benchmark(Description = "Projection query - 2 fields")] - public async Task SelectAsync_TwoFields() - { - var projections = await _store.SelectAsync( - d => new TwoFieldProjection { Id = d.Id, Name = d.Name }); - return projections.Count(); - } - - [Benchmark(Description = "Projection query - 4 fields")] - public async Task SelectAsync_FourFields() - { - var projections = await _store.SelectAsync( - d => new FourFieldProjection - { - Id = d.Id, - Name = d.Name, - Email = d.Email, - Category = d.Category - }); - return projections.Count(); - } - - [Benchmark(Description = "Projection query - nested field")] - public async Task SelectAsync_NestedField() - { - var projections = await _store.SelectAsync( - d => new NestedFieldProjection - { - Name = d.Name, - ContentTitle = d.Content.Title, - ContentAuthor = d.Content.Author - }); - return projections.Count(); - } - - [Benchmark(Description = "Projection query with filter - 2 fields")] - public async Task SelectAsync_WithFilter_TwoFields() - { - var projections = await _store.SelectAsync( - d => d.Category == "Category 5", - d => new TwoFieldProjection { Id = d.Id, Name = d.Name }); - return projections.Count(); - } - - [Benchmark(Description = "Full documents with filter (for comparison)")] - public async Task QueryAsync_WithFilter_FullDocuments() - { - var documents = await _store.QueryAsync( - d => d.Category == "Category 5"); - return documents.Count(); - } -} - -/// -/// Large document type with many fields to simulate realistic scenarios. -/// Approximately 2-3 KB per document when serialized. -/// -public class LargeDocument -{ - public string Id { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; - public string Email { get; set; } = string.Empty; - public string Description { get; set; } = string.Empty; - public string Category { get; set; } = string.Empty; - public List Tags { get; set; } = []; - public Dictionary Metadata { get; set; } = []; - public ContentBlock Content { get; set; } = new(); - public List Attachments { get; set; } = []; - public TimestampInfo Timestamps { get; set; } = new(); - public Statistics Stats { get; set; } = new(); -} - -public class ContentBlock -{ - public string Title { get; set; } = string.Empty; - public string Body { get; set; } = string.Empty; - public string Author { get; set; } = string.Empty; - public DateTime PublishDate { get; set; } - public int ViewCount { get; set; } - public int LikeCount { get; set; } -} - -public class Attachment -{ - public string FileName { get; set; } = string.Empty; - public long FileSize { get; set; } - public string MimeType { get; set; } = string.Empty; - public string Url { get; set; } = string.Empty; -} - -public class TimestampInfo -{ - public DateTime CreatedAt { get; set; } - public DateTime UpdatedAt { get; set; } - public DateTime LastAccessedAt { get; set; } -} - -public class Statistics -{ - public int TotalViews { get; set; } - public int UniqueVisitors { get; set; } - public TimeSpan AvgTimeOnPage { get; set; } - public double BounceRate { get; set; } -} - -// Projection DTOs -public class TwoFieldProjection -{ - public string Id { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; -} - -public class FourFieldProjection -{ - public string Id { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; - public string Email { get; set; } = string.Empty; - public string Category { get; set; } = string.Empty; -} - -public class NestedFieldProjection -{ - public string Name { get; set; } = string.Empty; - public string ContentTitle { get; set; } = string.Empty; - public string ContentAuthor { get; set; } = string.Empty; -} diff --git a/src/tests/LiteDocumentStore.Benchmarks/README.md b/src/tests/LiteDocumentStore.Benchmarks/README.md index d2113bd..0240e7a 100644 --- a/src/tests/LiteDocumentStore.Benchmarks/README.md +++ b/src/tests/LiteDocumentStore.Benchmarks/README.md @@ -17,9 +17,6 @@ dotnet run -c Release # Run comparison benchmarks (LiteDocumentStore vs Dapper vs LiteDB) dotnet run -c Release --filter *Comparison* -# Run projection query benchmarks -dotnet run -c Release --filter *ProjectionQuery* - # Run virtual column benchmarks dotnet run -c Release --filter *VirtualColumn* ``` @@ -55,79 +52,6 @@ BenchmarkDotNet automatically generates reports in `BenchmarkDotNet.Artifacts/re - Both SQLite-based solutions should significantly outperform LiteDB for bulk operations - LiteDB may be faster for single operations due to less serialization overhead -### 2. Projection Query Benchmark - -**Purpose**: Validates the claim that projection queries are 50-70% faster and use 80-90% less memory than retrieving full documents. - -**Scenarios Tested**: -1. **Baseline**: Full document retrieval (`GetAllAsync`) -2. **Two Field Projection**: Select only ID and Name -3. **Four Field Projection**: Select ID, Name, Email, Category -4. **Nested Field Projection**: Select fields including nested properties -5. **Filtered Projection**: Projection with WHERE clause -6. **Filtered Full Documents**: Full documents with WHERE clause (comparison) - -**Test Data**: -- 1,000 documents per test -- ~2-3 KB per document when serialized -- Realistic structure with nested objects, lists, and dictionaries - -### Expected Results - -**Design Claims**: 50-70% faster, 80-90% less memory - -**Actual Results (Validated)**: - -| Metric | Baseline (Full Docs) | Projection (2 fields) | Improvement | -|--------|---------------------|----------------------|-------------| -| **Time** | 5,496 μs (100%) | 555 μs (10%) | **90% faster** | -| **Memory** | 7.95 MB (100%) | 137 KB (1.7%) | **98% less** | - -### Interpreting Results - -**Time (Mean)**: -- Lower is better -- Projection queries should be significantly faster -- The improvement increases with larger documents and more fields omitted - -**Memory (Allocated)**: -- Lower is better -- Projection queries allocate much less memory -- Most savings come from not deserializing unused fields - -**Example Output**: -``` -| Method | Mean | Error | StdDev | Ratio | Gen0 | Allocated | Alloc Ratio | -|------------------------------------ |---------:|---------:|---------:|------:|--------:|----------:|------------:| -| GetAllAsync_FullDocuments | 45.23 ms | 0.892 ms | 0.835 ms | 1.00 | 1200.00 | 3.85 MB | 1.00 | -| SelectAsync_TwoFields | 15.87 ms | 0.312 ms | 0.292 ms | 0.35 | 125.00 | 0.42 MB | 0.11 | -| SelectAsync_FourFields | 18.45 ms | 0.365 ms | 0.341 ms | 0.41 | 187.50 | 0.65 MB | 0.17 | -| SelectAsync_NestedField | 19.12 ms | 0.378 ms | 0.353 ms | 0.42 | 175.00 | 0.58 MB | 0.15 | -| SelectAsync_WithFilter_TwoFields | 2.34 ms | 0.046 ms | 0.043 ms | 0.05 | 15.00 | 0.05 MB | 0.01 | -| QueryAsync_WithFilter_FullDocuments | 6.78 ms | 0.134 ms | 0.125 ms | 0.15 | 125.00 | 0.42 MB | 0.11 | -``` - -In this example: -- **Two field projection is 65% faster** (0.35 ratio = 35% of baseline time) -- **Memory usage is 89% lower** (0.11 ratio = 11% of baseline allocation) -- **Filtered projections show even better results** due to smaller result sets - -## Understanding the Results - -### Why Projections Are Faster - -1. **Less Data Transfer**: SQLite only returns selected fields via `json_extract()` -2. **Smaller JSON Payloads**: Less text to parse and deserialize -3. **Simpler Object Construction**: Projection DTOs are simpler than full documents -4. **Better Cache Locality**: Smaller objects fit better in CPU cache - -### Why Memory Usage Is Lower - -1. **No Unused Field Allocation**: Only projected fields are deserialized -2. **Smaller Collection Objects**: Lists/dictionaries that aren't projected aren't allocated -3. **Less String Interning**: Fewer string fields means less heap pressure -4. **Reduced GC Pressure**: Fewer objects means less work for garbage collector - ## Virtual Column Benchmark **Purpose**: Demonstrates the query performance improvements when using virtual (generated) columns with indexes on frequently queried JSON fields. diff --git a/src/tests/LiteDocumentStore.Benchmarks/SimplifiedComparisonBenchmark.cs b/src/tests/LiteDocumentStore.Benchmarks/SimplifiedComparisonBenchmark.cs index 67cdd77..e8c0ae2 100644 --- a/src/tests/LiteDocumentStore.Benchmarks/SimplifiedComparisonBenchmark.cs +++ b/src/tests/LiteDocumentStore.Benchmarks/SimplifiedComparisonBenchmark.cs @@ -196,7 +196,7 @@ public async Task RawDapper_FullScan() [Benchmark] public async Task LiteDocumentStore_QueryByCategory() { - var results = await _documentStore.QueryAsync(d => d.Category == "Category 5"); + var results = await _documentStore.QueryAsync("$.Category", "Category 5"); return results.Count(); } diff --git a/src/tests/LiteDocumentStore.Benchmarks/VirtualColumnBenchmark.cs b/src/tests/LiteDocumentStore.Benchmarks/VirtualColumnBenchmark.cs index 9a06f6f..ac5b2b2 100644 --- a/src/tests/LiteDocumentStore.Benchmarks/VirtualColumnBenchmark.cs +++ b/src/tests/LiteDocumentStore.Benchmarks/VirtualColumnBenchmark.cs @@ -127,42 +127,42 @@ public async Task Cleanup() [Benchmark(Baseline = true, Description = "Query by category WITHOUT virtual column")] public async Task Query_WithoutVirtualColumn_ByCategory() { - var results = await _storeWithoutVirtual.QueryAsync(p => p.Category == "Category 25"); + var results = await _storeWithoutVirtual.QueryAsync("$.Category", "Category 25"); return results.Count(); } [Benchmark(Description = "Query by category WITH virtual column and index")] public async Task Query_WithVirtualColumn_ByCategory() { - var results = await _storeWithVirtual.QueryAsync(p => p.Category == "Category 25"); + var results = await _storeWithVirtual.QueryAsync("$.Category", "Category 25"); return results.Count(); } [Benchmark(Description = "Query by SKU WITHOUT virtual column")] public async Task Query_WithoutVirtualColumn_BySku() { - var results = await _storeWithoutVirtual.QueryAsync(p => p.Sku == "SKU-005000"); + var results = await _storeWithoutVirtual.QueryAsync("$.Sku", "SKU-005000"); return results.Count(); } [Benchmark(Description = "Query by SKU WITH virtual column and index")] public async Task Query_WithVirtualColumn_BySku() { - var results = await _storeWithVirtual.QueryAsync(p => p.Sku == "SKU-005000"); + var results = await _storeWithVirtual.QueryAsync("$.Sku", "SKU-005000"); return results.Count(); } [Benchmark(Description = "Query nested property WITHOUT virtual column")] public async Task Query_WithoutVirtualColumn_NestedProperty() { - var results = await _storeWithoutVirtual.QueryAsync(p => p.Metadata.Brand == "Brand 42"); + var results = await _storeWithoutVirtual.QueryAsync("$.Metadata.Brand", "Brand 42"); return results.Count(); } [Benchmark(Description = "Query nested property WITH virtual column and index")] public async Task Query_WithVirtualColumn_NestedProperty() { - var results = await _storeWithVirtual.QueryAsync(p => p.Metadata.Brand == "Brand 42"); + var results = await _storeWithVirtual.QueryAsync("$.Metadata.Brand", "Brand 42"); return results.Count(); } diff --git a/src/tests/LiteDocumentStore.IntegrationTests/DatabaseSeederExamples.cs b/src/tests/LiteDocumentStore.IntegrationTests/DatabaseSeederExamples.cs index 34ae3d8..675c301 100644 --- a/src/tests/LiteDocumentStore.IntegrationTests/DatabaseSeederExamples.cs +++ b/src/tests/LiteDocumentStore.IntegrationTests/DatabaseSeederExamples.cs @@ -194,9 +194,20 @@ await store.AddVirtualColumnAsync( "age", createIndex: true); - // Assert - Virtual column should work with seeded data - var youngPersons = await store.QueryAsync(p => p.Age < 30); - var youngList = youngPersons.ToList(); + // Assert - Virtual column should work with seeded data. Range predicates are no longer + // expressible through the store API, so seek the indexed virtual column via raw SQL and + // load each matching document through the public GetAsync. + var youngIds = await store.Connection.QueryStringsAsync( + "SELECT id FROM [PersonEntity] WHERE age < @Age", + ("Age", 30)); + + var youngList = new List(); + foreach (var id in youngIds) + { + var person = await store.GetAsync(id!); + Assert.NotNull(person); + youngList.Add(person); + } Assert.NotEmpty(youngList); Assert.All(youngList, p => Assert.True(p.Age < 30)); diff --git a/src/tests/LiteDocumentStore.IntegrationTests/DocumentStoreIntegrationTests.cs b/src/tests/LiteDocumentStore.IntegrationTests/DocumentStoreIntegrationTests.cs index d41c63e..e5b6f22 100644 --- a/src/tests/LiteDocumentStore.IntegrationTests/DocumentStoreIntegrationTests.cs +++ b/src/tests/LiteDocumentStore.IntegrationTests/DocumentStoreIntegrationTests.cs @@ -1,4 +1,3 @@ -using Dapper; using Microsoft.Data.Sqlite; using Xunit; @@ -51,7 +50,7 @@ public async Task CreateTableAsync_CreatesTable() // Assert var checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name='Person'"; - var result = _store.Connection.QueryFirstOrDefault(checkSql); + var result = await _store.Connection.QueryFirstStringAsync(checkSql); Assert.Equal("Person", result); } @@ -562,7 +561,7 @@ public async Task IsHealthyAsync_ValidatesSqliteVersion() Assert.True(isHealthy); // Verify SQLite version is 3.45+ - var version = await _connection.QueryFirstOrDefaultAsync("SELECT sqlite_version()"); + var version = await _connection.QueryFirstStringAsync("SELECT sqlite_version()"); Assert.NotNull(version); Assert.True(Version.TryParse(version, out var sqliteVersion)); Assert.True(sqliteVersion >= new Version(3, 45, 0), @@ -599,7 +598,7 @@ public async Task CreateIndexAsync_CreatesIndex_OnJsonPath() // Assert var checkSql = "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_person_email'"; - var count = await _connection.QueryFirstOrDefaultAsync(checkSql); + var count = await _connection.ExecuteScalarAsync(checkSql); Assert.Equal(1, count); } @@ -614,7 +613,7 @@ public async Task CreateIndexAsync_WithAutoGeneratedName_CreatesIndex() // Assert - check that an index was created (name is auto-generated) var checkSql = "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name LIKE 'idx_Person_name%'"; - var count = await _connection.QueryFirstOrDefaultAsync(checkSql); + var count = await _connection.ExecuteScalarAsync(checkSql); Assert.Equal(1, count); } @@ -630,7 +629,7 @@ public async Task CreateIndexAsync_IndexAlreadyExists_DoesNotThrow() // Assert var checkSql = "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_person_email'"; - var count = await _connection.QueryFirstOrDefaultAsync(checkSql); + var count = await _connection.ExecuteScalarAsync(checkSql); Assert.Equal(1, count); } @@ -645,7 +644,7 @@ public async Task CreateIndexAsync_NestedProperty_CreatesIndex() // Assert var checkSql = "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_person_address_city'"; - var count = await _connection.QueryFirstOrDefaultAsync(checkSql); + var count = await _connection.ExecuteScalarAsync(checkSql); Assert.Equal(1, count); } @@ -666,7 +665,7 @@ await _store.CreateCompositeIndexAsync( // Assert var checkSql = "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_person_name_age'"; - var count = await _connection.QueryFirstOrDefaultAsync(checkSql); + var count = await _connection.ExecuteScalarAsync(checkSql); Assert.Equal(1, count); } @@ -686,7 +685,7 @@ await _store.CreateCompositeIndexAsync( // Assert - check that a composite index was created var checkSql = "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name LIKE 'idx_Person_composite_%'"; - var count = await _connection.QueryFirstOrDefaultAsync(checkSql); + var count = await _connection.ExecuteScalarAsync(checkSql); Assert.Equal(1, count); } @@ -714,7 +713,7 @@ await _store.CreateCompositeIndexAsync( // Assert var checkSql = "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_person_name_age'"; - var count = await _connection.QueryFirstOrDefaultAsync(checkSql); + var count = await _connection.ExecuteScalarAsync(checkSql); Assert.Equal(1, count); } @@ -740,11 +739,22 @@ public async Task CreateIndexAsync_ImprovesQueryPerformance() // Assert - Verify the index exists and can be used // Note: JSON path uses PascalCase to match default System.Text.Json serialization - var queryPlan = await _connection.QueryAsync( - "EXPLAIN QUERY PLAN SELECT json(data) FROM Person WHERE json_extract(data, '$.Email') = 'person50@example.com'"); + // EXPLAIN QUERY PLAN yields a 'detail' column per step; read it by ordinal. + var planParts = new List(); + await using (var command = _connection.CreateCommand()) + { + command.CommandText = + "EXPLAIN QUERY PLAN SELECT json(data) FROM Person WHERE json_extract(data, '$.Email') = 'person50@example.com'"; + await using var reader = await command.ExecuteReaderAsync(); + var detailOrdinal = reader.GetOrdinal("detail"); + while (await reader.ReadAsync()) + { + planParts.Add(reader.GetString(detailOrdinal)); + } + } // The query plan should mention the index - var planText = string.Join(" ", queryPlan.Select(p => p.detail)); + var planText = string.Join(" ", planParts); Assert.Contains("idx_", planText.ToLower()); } @@ -794,131 +804,6 @@ public async Task QueryAsync_WithJsonPath_NoMatches_ReturnsEmpty() Assert.Empty(results); } - [Fact] - public async Task QueryAsync_WithPredicate_SimpleEquality_ReturnsMatchingDocuments() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - await _store.UpsertAsync("p2", new Person { Name = "Jane Smith", Age = 25, Email = "jane@example.com" }); - - // Act - var results = await _store.QueryAsync(p => p.Name == "Jane Smith"); - - // Assert - var resultList = results.ToList(); - Assert.Single(resultList); - Assert.Equal("Jane Smith", resultList[0].Name); - Assert.Equal(25, resultList[0].Age); - } - - [Fact] - public async Task QueryAsync_WithPredicate_Comparison_ReturnsMatchingDocuments() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - await _store.UpsertAsync("p2", new Person { Name = "Jane Smith", Age = 25, Email = "jane@example.com" }); - await _store.UpsertAsync("p3", new Person { Name = "Bob Johnson", Age = 35, Email = "bob@example.com" }); - - // Act - var results = await _store.QueryAsync(p => p.Age > 28); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, p => p.Name == "John Doe"); - Assert.Contains(resultList, p => p.Name == "Bob Johnson"); - } - - [Fact] - public async Task QueryAsync_WithPredicate_AndCondition_ReturnsMatchingDocuments() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - await _store.UpsertAsync("p2", new Person { Name = "Jane Smith", Age = 30, Email = "jane@example.com" }); - await _store.UpsertAsync("p3", new Person { Name = "Bob Johnson", Age = 25, Email = "bob@example.com" }); - - // Act - var results = await _store.QueryAsync(p => p.Age == 30 && p.Name.StartsWith("J")); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, p => p.Name == "John Doe"); - Assert.Contains(resultList, p => p.Name == "Jane Smith"); - } - - [Fact] - public async Task QueryAsync_WithPredicate_OrCondition_ReturnsMatchingDocuments() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - await _store.UpsertAsync("p2", new Person { Name = "Jane Smith", Age = 25, Email = "jane@example.com" }); - await _store.UpsertAsync("p3", new Person { Name = "Bob Johnson", Age = 35, Email = "bob@example.com" }); - - // Act - var results = await _store.QueryAsync(p => p.Age < 27 || p.Age > 32); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, p => p.Name == "Jane Smith"); - Assert.Contains(resultList, p => p.Name == "Bob Johnson"); - } - - [Fact] - public async Task QueryAsync_WithPredicate_StringContains_ReturnsMatchingDocuments() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - await _store.UpsertAsync("p2", new Person { Name = "Jane Smith", Age = 25, Email = "jane@test.com" }); - await _store.UpsertAsync("p3", new Person { Name = "Bob Johnson", Age = 35, Email = "bob@example.com" }); - - // Act - var results = await _store.QueryAsync(p => p.Email.Contains("@example.com")); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, p => p.Name == "John Doe"); - Assert.Contains(resultList, p => p.Name == "Bob Johnson"); - } - - [Fact] - public async Task QueryAsync_WithPredicate_NestedProperty_ReturnsMatchingDocuments() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new PersonWithAddress - { - Name = "John Doe", - Address = new Address { City = "New York", Street = "5th Ave", Country = "USA" } - }); - await _store.UpsertAsync("p2", new PersonWithAddress - { - Name = "Jane Smith", - Address = new Address { City = "London", Street = "Baker St", Country = "UK" } - }); - await _store.UpsertAsync("p3", new PersonWithAddress - { - Name = "Bob Johnson", - Address = new Address { City = "New York", Street = "Broadway", Country = "USA" } - }); - - // Act - var results = await _store.QueryAsync(p => p.Address.City == "New York"); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, p => p.Name == "John Doe"); - Assert.Contains(resultList, p => p.Name == "Bob Johnson"); - } - [Fact] public async Task QueryAsync_WithJsonPath_NullOrWhitespace_ThrowsArgumentException() { @@ -930,154 +815,6 @@ await Assert.ThrowsAsync(async () => await _store.QueryAsync("", 30)); } - [Fact] - public async Task QueryAsync_WithPredicate_NullPredicate_ThrowsArgumentNullException() - { - // Arrange - await _store.CreateTableAsync(); - - // Act & Assert - await Assert.ThrowsAsync(async () => - await _store.QueryAsync(null!)); - } - - [Fact] - public async Task SelectAsync_WithSelector_ReturnsProjectedFields() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - await _store.UpsertAsync("p2", new Person { Name = "Jane Smith", Age = 25, Email = "jane@example.com" }); - await _store.UpsertAsync("p3", new Person { Name = "Bob Johnson", Age = 35, Email = "bob@example.com" }); - - // Act - Select only Name and Email fields - var results = await _store.SelectAsync( - p => new PersonProjection { Name = p.Name, Email = p.Email }); - - // Assert - var resultList = results.ToList(); - Assert.Equal(3, resultList.Count); - Assert.Contains(resultList, r => r.Name == "John Doe" && r.Email == "john@example.com"); - Assert.Contains(resultList, r => r.Name == "Jane Smith" && r.Email == "jane@example.com"); - Assert.Contains(resultList, r => r.Name == "Bob Johnson" && r.Email == "bob@example.com"); - } - - [Fact] - public async Task SelectAsync_WithPredicateAndSelector_ReturnsFilteredProjectedFields() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - await _store.UpsertAsync("p2", new Person { Name = "Jane Smith", Age = 25, Email = "jane@example.com" }); - await _store.UpsertAsync("p3", new Person { Name = "Bob Johnson", Age = 35, Email = "bob@example.com" }); - - // Act - Select Name and Email from people over 28 - var results = await _store.SelectAsync( - p => p.Age > 28, - p => new PersonProjection { Name = p.Name, Email = p.Email }); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, r => r.Name == "John Doe" && r.Email == "john@example.com"); - Assert.Contains(resultList, r => r.Name == "Bob Johnson" && r.Email == "bob@example.com"); - Assert.DoesNotContain(resultList, r => r.Name == "Jane Smith"); - } - - [Fact] - public async Task SelectAsync_WithAnonymousType_ReturnsProjectedFields() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - - // Act - Use anonymous type for projection - var results = await _store.SelectAsync( - p => new { p.Name, p.Age }); - - // Assert - var resultList = results.ToList(); - Assert.Single(resultList); - Assert.Equal("John Doe", resultList[0].Name); - Assert.Equal(30, resultList[0].Age); - } - - [Fact] - public async Task SelectAsync_WithNestedProperties_ReturnsProjectedFields() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new PersonWithAddress - { - Name = "John Doe", - Address = new Address { City = "New York", Street = "5th Avenue", Country = "USA" } - }); - await _store.UpsertAsync("p2", new PersonWithAddress - { - Name = "Jane Smith", - Address = new Address { City = "London", Street = "Baker St", Country = "UK" } - }); - - // Act - Select nested property - var results = await _store.SelectAsync( - p => new PersonCityProjection { Name = p.Name, City = p.Address.City }); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, r => r.Name == "John Doe" && r.City == "New York"); - Assert.Contains(resultList, r => r.Name == "Jane Smith" && r.City == "London"); - } - - [Fact] - public async Task SelectAsync_EmptyTable_ReturnsEmptyResults() - { - // Arrange - await _store.CreateTableAsync(); - - // Act - var results = await _store.SelectAsync( - p => new PersonProjection { Name = p.Name, Email = p.Email }); - - // Assert - Assert.Empty(results); - } - - [Fact] - public async Task SelectAsync_VerifiesOnlySelectedFieldsRetrieved() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Person { Name = "John Doe", Age = 30, Email = "john@example.com" }); - - // Act - Select only Name field (not Age) - var results = await _store.SelectAsync( - p => new NameOnlyProjection { Name = p.Name }); - - // Assert - var result = results.Single(); - Assert.Equal("John Doe", result.Name); - // Verify we can use the projected result without the other fields - Assert.NotNull(result); - } - - private class PersonProjection - { - public string Name { get; set; } = string.Empty; - public string Email { get; set; } = string.Empty; - } - - private class PersonCityProjection - { - public string Name { get; set; } = string.Empty; - public string City { get; set; } = string.Empty; - } - - private class NameOnlyProjection - { - public string Name { get; set; } = string.Empty; - } - private class Person { public string Name { get; set; } = string.Empty; diff --git a/src/tests/LiteDocumentStore.IntegrationTests/ExceptionIntegrationTests.cs b/src/tests/LiteDocumentStore.IntegrationTests/ExceptionIntegrationTests.cs index 715ec07..5b9cc31 100644 --- a/src/tests/LiteDocumentStore.IntegrationTests/ExceptionIntegrationTests.cs +++ b/src/tests/LiteDocumentStore.IntegrationTests/ExceptionIntegrationTests.cs @@ -2,7 +2,6 @@ using Microsoft.Data.Sqlite; using System.Text.Json; using Xunit; -using Dapper; namespace LiteDocumentStore.IntegrationTests; @@ -53,7 +52,7 @@ public async Task SerializationException_ThrownWhenDeserializationFails() // Manually insert invalid JSON await _connection.ExecuteAsync( "INSERT INTO [StrictModel] (id, data) VALUES (@Id, jsonb(@Data))", - new { Id = "test-1", Data = "{\"RequiredInt\": \"not-a-number\"}" }); + ("Id", "test-1"), ("Data", "{\"RequiredInt\": \"not-a-number\"}")); // Act & Assert - Deserialization should fail because "not-a-number" cannot be parsed as int var exception = await Assert.ThrowsAnyAsync( diff --git a/src/tests/LiteDocumentStore.IntegrationTests/LiteDocumentStore.IntegrationTests.csproj b/src/tests/LiteDocumentStore.IntegrationTests/LiteDocumentStore.IntegrationTests.csproj index af9c94b..c04ffd0 100644 --- a/src/tests/LiteDocumentStore.IntegrationTests/LiteDocumentStore.IntegrationTests.csproj +++ b/src/tests/LiteDocumentStore.IntegrationTests/LiteDocumentStore.IntegrationTests.csproj @@ -10,11 +10,11 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/tests/LiteDocumentStore.IntegrationTests/MigrationIntegrationTests.cs b/src/tests/LiteDocumentStore.IntegrationTests/MigrationIntegrationTests.cs index 9a9873c..e99d119 100644 --- a/src/tests/LiteDocumentStore.IntegrationTests/MigrationIntegrationTests.cs +++ b/src/tests/LiteDocumentStore.IntegrationTests/MigrationIntegrationTests.cs @@ -1,3 +1,4 @@ +using LiteDocumentStore.Exceptions; using Microsoft.Data.Sqlite; using Xunit; @@ -298,4 +299,51 @@ public async Task MigrationHistoryRecord_ContainsAppliedAt_Timestamp() Assert.True(record.AppliedAt >= before); Assert.True(record.AppliedAt <= after); } + + [Fact] + public async Task ApplyMigrationAsync_VersionBelowCurrentMax_IsSkipped() + { + // Arrange - apply v1 and v3, leaving a gap at v2. The runner tracks the highest applied + // version, so a later-introduced v2 sits below the current max. + await _runner.ApplyMigrationAsync(new Migration(1, "V1", "SELECT 1", "SELECT 1")); + await _runner.ApplyMigrationAsync(new Migration(3, "V3", "SELECT 1", "SELECT 1")); + + // Act - v2 (below current max version 3) is skipped by the linear version model + var v2 = new Migration( + version: 2, + name: "V2", + upSql: "CREATE TABLE ShouldNotExist (id TEXT PRIMARY KEY)", + downSql: "DROP TABLE ShouldNotExist"); + var applied = await _runner.ApplyMigrationAsync(v2); + + // Assert - not applied, its Up SQL never ran, and history is unchanged (only v1, v3) + Assert.False(applied); + + var introspector = new SchemaIntrospector(_connection); + Assert.False(await introspector.TableExistsAsync("ShouldNotExist")); + + var versions = (await _runner.GetAppliedMigrationsAsync()).Select(m => m.Version).ToList(); + Assert.Equal(new long[] { 1, 3 }, versions); + } + + [Fact] + public async Task RollbackToVersionAsync_MissingDefinition_Throws_AndRollsBackNothing() + { + // Arrange - apply v1 and v2 + var v1 = new Migration(1, "V1", "CREATE TABLE T1 (id TEXT PRIMARY KEY)", "DROP TABLE T1"); + var v2 = new Migration(2, "V2", "CREATE TABLE T2 (id TEXT PRIMARY KEY)", "DROP TABLE T2"); + await _runner.ApplyMigrationAsync(v1); + await _runner.ApplyMigrationAsync(v2); + + // Act & Assert - rolling back to 0 requires both definitions; only v2 is supplied, so the + // whole operation must fail before mutating anything (no partial rollback). + await Assert.ThrowsAsync(async () => + await _runner.RollbackToVersionAsync(0, new[] { v2 })); + + // Nothing was rolled back: both tables remain and both versions stay recorded. + var introspector = new SchemaIntrospector(_connection); + Assert.True(await introspector.TableExistsAsync("T1")); + Assert.True(await introspector.TableExistsAsync("T2")); + Assert.Equal(2, await _runner.GetCurrentVersionAsync()); + } } diff --git a/src/tests/LiteDocumentStore.IntegrationTests/SchemaIntrospectionIntegrationTests.cs b/src/tests/LiteDocumentStore.IntegrationTests/SchemaIntrospectionIntegrationTests.cs index 4aa0322..8d54c06 100644 --- a/src/tests/LiteDocumentStore.IntegrationTests/SchemaIntrospectionIntegrationTests.cs +++ b/src/tests/LiteDocumentStore.IntegrationTests/SchemaIntrospectionIntegrationTests.cs @@ -50,6 +50,22 @@ public async Task GetTablesAsync_WithCreatedTables_ReturnsAllTables() Assert.Contains(tables, t => t.Name == "Order"); } + [Fact] + public async Task GetColumnsAsync_TableNameWithClosingBracket_IsEscaped() + { + // Arrange - a table whose name contains ']' would break out of the [ ] identifier quoting + // in the PRAGMA statement if the bracket were not escaped. + await _connection.ExecuteAsync( + "CREATE TABLE \"weird]name\" (id TEXT PRIMARY KEY, val TEXT)"); + + // Act + var columns = (await _introspector.GetColumnsAsync("weird]name")).ToList(); + + // Assert - the PRAGMA resolved the real table rather than erroring or hitting the wrong one + Assert.Contains(columns, c => c.Name == "id"); + Assert.Contains(columns, c => c.Name == "val"); + } + [Fact] public async Task TableExistsAsync_WithExistingTable_ReturnsTrue() { diff --git a/src/tests/LiteDocumentStore.IntegrationTests/VirtualColumnIntegrationTests.cs b/src/tests/LiteDocumentStore.IntegrationTests/VirtualColumnIntegrationTests.cs index 3a94284..98152a4 100644 --- a/src/tests/LiteDocumentStore.IntegrationTests/VirtualColumnIntegrationTests.cs +++ b/src/tests/LiteDocumentStore.IntegrationTests/VirtualColumnIntegrationTests.cs @@ -1,4 +1,3 @@ -using Dapper; using Microsoft.Data.Sqlite; using Xunit; @@ -6,7 +5,7 @@ namespace LiteDocumentStore.IntegrationTests; /// /// Integration tests for virtual column functionality. -/// Tests cover virtual column creation, caching, and query optimization. +/// Tests cover virtual column creation and query optimization via the generated columns. /// public class VirtualColumnIntegrationTests : IDisposable { @@ -39,16 +38,38 @@ public void Dispose() } } + // Seeks the given WHERE clause against the Product table (typically hitting a virtual + // column) and loads each matching document through the public store API. Range/string + // predicates are no longer expressible through the store query API, so tests exercise the + // virtual columns via raw SQL and rehydrate documents by id. + private async Task> LoadProductsAsync( + string whereSql, + params (string Name, object? Value)[] parameters) + { + var ids = await _connection.QueryStringsAsync( + $"SELECT id FROM Product WHERE {whereSql}", parameters); + + var products = new List(); + foreach (var id in ids) + { + var product = await _store.GetAsync(id!); + Assert.NotNull(product); + products.Add(product); + } + + return products; + } + #region Virtual Column Creation Tests [Fact] public async Task AddVirtualColumnAsync_Debug_SQLiteSyntax() { - // Debug test to verify SQLite virtual column syntax works + // Debug test to verify SQLite generated (virtual) column syntax works using var memConnection = new SqliteConnection("Data Source=:memory:"); await memConnection.OpenAsync(); - var version = await memConnection.QueryFirstAsync("SELECT sqlite_version()"); + var version = await memConnection.QueryFirstStringAsync("SELECT sqlite_version()"); await memConnection.ExecuteAsync(@" CREATE TABLE Test1 ( @@ -58,13 +79,14 @@ CREATE TABLE Test1 ( c INTEGER GENERATED ALWAYS AS (a + b) )"); - var cols1 = (await memConnection.QueryAsync("PRAGMA table_xinfo(Test1)")).ToList(); - var colNames = cols1.Select(c => (string)c.name).ToList(); + var introspector = new SchemaIntrospector(memConnection); + var columns = await introspector.GetColumnsAsync("Test1"); + var colNames = columns.Select(c => c.Name).ToList(); await memConnection.ExecuteAsync("INSERT INTO Test1 (id, a, b) VALUES (1, 10, 20)"); - var result = await memConnection.QueryFirstOrDefaultAsync("SELECT id, a, b, c FROM Test1 WHERE id = 1"); + var generated = await memConnection.ExecuteScalarAsync("SELECT c FROM Test1 WHERE id = 1"); - Assert.True(colNames.Contains("c") || result?.c != null, + Assert.True(colNames.Contains("c") || generated == 30, $"Generated column 'c' not found. Columns: [{string.Join(", ", colNames)}]. SQLite version: {version}"); } @@ -155,20 +177,19 @@ public async Task AddVirtualColumnAsync_VirtualColumnValues_AreCorrect() await _store.AddVirtualColumnAsync(x => x.Price, "price", columnType: "REAL"); // Assert - Query using virtual columns directly - var results = await _connection.QueryAsync( - "SELECT id, category, price FROM Product WHERE category = 'Electronics' ORDER BY price"); - var resultList = results.ToList(); + var categories = await _connection.QueryStringsAsync( + "SELECT category FROM Product WHERE category = 'Electronics' ORDER BY price"); - Assert.Equal(2, resultList.Count); - Assert.Equal("Electronics", (string)resultList[0].category); + Assert.Equal(2, categories.Count); + Assert.Equal("Electronics", categories[0]); } #endregion - #region QueryAsync with Virtual Columns Tests + #region Virtual Column Querying Tests (raw SQL seeks over the generated columns) [Fact] - public async Task QueryAsync_WithVirtualColumn_UsesVirtualColumnInSql() + public async Task VirtualColumn_EqualitySeek_ReturnsMatchingDocuments() { // Arrange await _store.CreateTableAsync(); @@ -176,19 +197,18 @@ public async Task QueryAsync_WithVirtualColumn_UsesVirtualColumnInSql() await _store.UpsertAsync("p2", new Product { Name = "Gadget", Category = "Toys", Price = 49.99m }); await _store.UpsertAsync("p3", new Product { Name = "Tool", Category = "Hardware", Price = 19.99m }); - // Add virtual column await _store.AddVirtualColumnAsync(p => p.Category, "category", createIndex: true); - // Act - QueryAsync should now use the virtual column - var results = await _store.QueryAsync(p => p.Category == "Electronics"); + // Act + var results = await LoadProductsAsync("category = @Category", ("Category", "Electronics")); // Assert Assert.Single(results); - Assert.Equal("Widget", results.First().Name); + Assert.Equal("Widget", results[0].Name); } [Fact] - public async Task QueryAsync_WithVirtualColumn_SupportsRangeQueries() + public async Task VirtualColumn_RangeSeek_ReturnsMatchingDocuments() { // Arrange await _store.CreateTableAsync(); @@ -196,20 +216,19 @@ public async Task QueryAsync_WithVirtualColumn_SupportsRangeQueries() await _store.UpsertAsync("p2", new Product { Name = "Medium", Category = "B", Price = 50.00m }); await _store.UpsertAsync("p3", new Product { Name = "Expensive", Category = "C", Price = 100.00m }); - // Add virtual column for Price await _store.AddVirtualColumnAsync(p => p.Price, "price", createIndex: true, columnType: "REAL"); - // Act - QueryAsync should use virtual column for range query - var results = await _store.QueryAsync(p => p.Price > 30); + // Act + var results = await LoadProductsAsync("price > @Price", ("Price", 30.0)); // Assert - Assert.Equal(2, results.Count()); + Assert.Equal(2, results.Count); Assert.Contains(results, p => p.Name == "Medium"); Assert.Contains(results, p => p.Name == "Expensive"); } [Fact] - public async Task QueryAsync_WithVirtualColumn_SupportsStringMethods() + public async Task VirtualColumn_LikeSeek_ReturnsMatchingDocuments() { // Arrange await _store.CreateTableAsync(); @@ -217,19 +236,18 @@ public async Task QueryAsync_WithVirtualColumn_SupportsStringMethods() await _store.UpsertAsync("p2", new Product { Name = "Widget Basic", Category = "Electronics", Price = 19.99m }); await _store.UpsertAsync("p3", new Product { Name = "Gadget", Category = "Toys", Price = 9.99m }); - // Add virtual column await _store.AddVirtualColumnAsync(p => p.Name, "name_vc", createIndex: true); - // Act - QueryAsync with StartsWith - var results = await _store.QueryAsync(p => p.Name.StartsWith("Widget")); + // Act + var results = await LoadProductsAsync("name_vc LIKE @Pattern", ("Pattern", "Widget%")); // Assert - Assert.Equal(2, results.Count()); + Assert.Equal(2, results.Count); Assert.All(results, p => Assert.StartsWith("Widget", p.Name)); } [Fact] - public async Task QueryAsync_WithMultipleVirtualColumns_UsesAllColumns() + public async Task VirtualColumn_MultipleColumnSeek_ReturnsMatchingDocuments() { // Arrange await _store.CreateTableAsync(); @@ -237,81 +255,16 @@ public async Task QueryAsync_WithMultipleVirtualColumns_UsesAllColumns() await _store.UpsertAsync("p2", new Product { Name = "Gadget", Category = "Electronics", Price = 49.99m }); await _store.UpsertAsync("p3", new Product { Name = "Tool", Category = "Hardware", Price = 19.99m }); - // Add multiple virtual columns await _store.AddVirtualColumnAsync(p => p.Category, "category", createIndex: true); await _store.AddVirtualColumnAsync(p => p.Price, "price", createIndex: true, columnType: "REAL"); - // Act - Query with both conditions - var results = await _store.QueryAsync(p => p.Category == "Electronics" && p.Price > 30); - - // Assert - Assert.Single(results); - Assert.Equal("Gadget", results.First().Name); - } - - #endregion - - #region Virtual Column Cache Tests - - [Fact] - public async Task QueryAsync_LoadsVirtualColumnsFromSchema() - { - // Arrange - Create virtual column with initial store - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Product { Name = "Test", Category = "TestCat", Price = 10.00m }); - await _store.AddVirtualColumnAsync(p => p.Name, "name_vc", createIndex: true); - - // Create a NEW store instance (simulating app restart) - using var newStore = new DocumentStore(_connection); - - // Act - Query should discover and use the virtual column from schema - var results = await newStore.QueryAsync(p => p.Name == "Test"); - - // Assert - Assert.Single(results); - Assert.Equal("TestCat", results.First().Category); - } - - [Fact] - public async Task QueryAsync_CacheUpdatedWhenVirtualColumnAdded() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new Product { Name = "Widget", Category = "Electronics", Price = 29.99m }); - - // First query - no virtual columns, uses json_extract - var results1 = await _store.QueryAsync(p => p.Category == "Electronics"); - Assert.Single(results1); - - // Add virtual column - await _store.AddVirtualColumnAsync(p => p.Category, "category", createIndex: true); - - // Second query - should now use virtual column - var results2 = await _store.QueryAsync(p => p.Category == "Electronics"); - Assert.Single(results2); - } - - [Fact] - public async Task QueryAsync_NestedVirtualColumn_IsDiscoveredFromSchema() - { - // Arrange - await _store.CreateTableAsync(); - await _store.UpsertAsync("p1", new ProductWithMetadata - { - Name = "Widget", - Metadata = new ProductMetadata { Brand = "Acme", Country = "USA" } - }); - await _store.AddVirtualColumnAsync(p => p.Metadata.Brand, "brand"); - - // Create new store - using var newStore = new DocumentStore(_connection); - // Act - var results = await newStore.QueryAsync(p => p.Metadata.Brand == "Acme"); + var results = await LoadProductsAsync( + "category = @Category AND price > @Price", ("Category", "Electronics"), ("Price", 30.0)); // Assert Assert.Single(results); - Assert.Equal("Widget", results.First().Name); + Assert.Equal("Gadget", results[0].Name); } #endregion @@ -336,16 +289,16 @@ public async Task AddVirtualColumnAsync_IndexCanBeUsed_InRawQuery() await _store.AddVirtualColumnAsync(x => x.Category, "category", createIndex: true); // Act - Query using index - var result = await _connection.QueryFirstOrDefaultAsync( - "SELECT * FROM Product WHERE category = @Category", - new { Category = "Category 5" }); + var match = await _connection.QueryFirstStringAsync( + "SELECT id FROM Product WHERE category = @Category", + ("Category", "Category 5")); // Assert - Assert.NotNull(result); + Assert.NotNull(match); } [Fact] - public async Task QueryAsync_WithIndex_ReturnsCorrectResults() + public async Task VirtualColumn_IndexedSeek_ReturnsCorrectResults() { // Arrange await _store.CreateTableAsync(); @@ -367,10 +320,10 @@ public async Task QueryAsync_WithIndex_ReturnsCorrectResults() await _store.AddVirtualColumnAsync(p => p.Category, "category", createIndex: true); // Act - var results = await _store.QueryAsync(p => p.Category == "Target"); + var results = await LoadProductsAsync("category = @Category", ("Category", "Target")); // Assert - Assert.Equal(expectedProducts.Count, results.Count()); + Assert.Equal(expectedProducts.Count, results.Count); Assert.All(results, p => Assert.Equal("Target", p.Category)); } diff --git a/src/tests/LiteDocumentStore.IntegrationTests/WalConcurrencyIntegrationTests.cs b/src/tests/LiteDocumentStore.IntegrationTests/WalConcurrencyIntegrationTests.cs index 2e9dd3b..5bb971d 100644 --- a/src/tests/LiteDocumentStore.IntegrationTests/WalConcurrencyIntegrationTests.cs +++ b/src/tests/LiteDocumentStore.IntegrationTests/WalConcurrencyIntegrationTests.cs @@ -1,4 +1,3 @@ -using Dapper; using Microsoft.Data.Sqlite; using Xunit; @@ -58,7 +57,7 @@ public async Task WalMode_EnablesSuccessfully() await store.CreateTableAsync(); // Assert - Verify WAL mode is enabled - var journalMode = store.Connection.QueryFirstOrDefault("PRAGMA journal_mode"); + var journalMode = await store.Connection.QueryFirstStringAsync("PRAGMA journal_mode"); Assert.Equal("wal", journalMode, StringComparer.OrdinalIgnoreCase); // Verify WAL file is created diff --git a/src/tests/LiteDocumentStore.UnitTests/DocumentStoreTests.cs b/src/tests/LiteDocumentStore.UnitTests/DocumentStoreTests.cs index 8c717ad..1e06f1a 100644 --- a/src/tests/LiteDocumentStore.UnitTests/DocumentStoreTests.cs +++ b/src/tests/LiteDocumentStore.UnitTests/DocumentStoreTests.cs @@ -1,4 +1,3 @@ -using Dapper; using Xunit; namespace LiteDocumentStore.UnitTests; @@ -51,7 +50,7 @@ public async Task GetTableName_ReturnsTypeName() // Assert - verify table exists with correct name var checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name='TestPerson'"; - var result = store.Connection.QueryFirstOrDefault(checkSql); + var result = await store.Connection.QueryFirstStringAsync(checkSql); Assert.Equal("TestPerson", result); } @@ -220,193 +219,6 @@ private class TestPerson public string Email { get; set; } = string.Empty; } - private class TestCustomer - { - public string Id { get; set; } = Guid.NewGuid().ToString(); - public string Name { get; set; } = string.Empty; - public string Email { get; set; } = string.Empty; - public int Age { get; set; } - public bool Active { get; set; } - } - - private class CustomerProjection - { - public string Name { get; set; } = string.Empty; - public string Email { get; set; } = string.Empty; - } - - [Fact] - public async Task SelectAsync_WithSelector_ReturnsProjectedFields() - { - // Arrange - var testDbPath = Path.Combine(Path.GetTempPath(), $"test_{Guid.NewGuid()}.db"); - var options = new DocumentStoreOptions { ConnectionString = $"Data Source={testDbPath}" }; - var connectionFactory = new DefaultConnectionFactory(); - - using (var connection = connectionFactory.CreateConnection(options)) - { - var store = new DocumentStore(connection); - await store.CreateTableAsync(); - - // Insert test data - await store.UpsertAsync("1", new TestCustomer - { - Id = "1", - Name = "John Doe", - Email = "john@example.com", - Age = 30, - Active = true - }); - await store.UpsertAsync("2", new TestCustomer - { - Id = "2", - Name = "Jane Smith", - Email = "jane@example.com", - Age = 25, - Active = true - }); - - // Act - Select only Name and Email - var results = await store.SelectAsync( - x => new CustomerProjection { Name = x.Name, Email = x.Email }); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, r => r.Name == "John Doe" && r.Email == "john@example.com"); - Assert.Contains(resultList, r => r.Name == "Jane Smith" && r.Email == "jane@example.com"); - } - - // Cleanup - if (File.Exists(testDbPath)) - { - try { File.Delete(testDbPath); } catch { } - } - } - - [Fact] - public async Task SelectAsync_WithPredicateAndSelector_ReturnsFilteredProjectedFields() - { - // Arrange - var testDbPath = Path.Combine(Path.GetTempPath(), $"test_{Guid.NewGuid()}.db"); - var options = new DocumentStoreOptions { ConnectionString = $"Data Source={testDbPath}" }; - var connectionFactory = new DefaultConnectionFactory(); - - using (var connection = connectionFactory.CreateConnection(options)) - { - var store = new DocumentStore(connection); - await store.CreateTableAsync(); - - // Insert test data - await store.UpsertAsync("1", new TestCustomer - { - Id = "1", - Name = "John Doe", - Email = "john@example.com", - Age = 30, - Active = true - }); - await store.UpsertAsync("2", new TestCustomer - { - Id = "2", - Name = "Jane Smith", - Email = "jane@example.com", - Age = 25, - Active = false - }); - await store.UpsertAsync("3", new TestCustomer - { - Id = "3", - Name = "Bob Johnson", - Email = "bob@example.com", - Age = 35, - Active = true - }); - - // Act - Select Name and Email from only active customers - var results = await store.SelectAsync( - x => x.Active == true, - x => new CustomerProjection { Name = x.Name, Email = x.Email }); - - // Assert - var resultList = results.ToList(); - Assert.Equal(2, resultList.Count); - Assert.Contains(resultList, r => r.Name == "John Doe"); - Assert.Contains(resultList, r => r.Name == "Bob Johnson"); - Assert.DoesNotContain(resultList, r => r.Name == "Jane Smith"); - } - - // Cleanup - if (File.Exists(testDbPath)) - { - try { File.Delete(testDbPath); } catch { } - } - } - - [Fact] - public async Task SelectAsync_WithAnonymousType_ReturnsProjectedFields() - { - // Arrange - var testDbPath = Path.Combine(Path.GetTempPath(), $"test_{Guid.NewGuid()}.db"); - var options = new DocumentStoreOptions { ConnectionString = $"Data Source={testDbPath}" }; - var connectionFactory = new DefaultConnectionFactory(); - - using (var connection = connectionFactory.CreateConnection(options)) - { - var store = new DocumentStore(connection); - await store.CreateTableAsync(); - - // Insert test data - await store.UpsertAsync("1", new TestCustomer - { - Id = "1", - Name = "John Doe", - Email = "john@example.com", - Age = 30, - Active = true - }); - - // Act - Select using anonymous type - var results = await store.SelectAsync( - x => new { x.Name, x.Email }); - - // Assert - var resultList = results.ToList(); - Assert.Single(resultList); - Assert.Equal("John Doe", resultList[0].Name); - Assert.Equal("john@example.com", resultList[0].Email); - } - - // Cleanup - if (File.Exists(testDbPath)) - { - try { File.Delete(testDbPath); } catch { } - } - } - - [Fact] - public async Task SelectAsync_WithNullSelector_ThrowsArgumentNullException() - { - // Arrange - var options = new DocumentStoreOptions { ConnectionString = $"Data Source={_testDbPath}" }; - var connectionFactory = new DefaultConnectionFactory(); - - using (var connection = connectionFactory.CreateConnection(options)) - { - var store = new DocumentStore(connection); - - // Act & Assert - await Assert.ThrowsAsync(async () => - await store.SelectAsync(null!)); - } - - // Cleanup - if (File.Exists(_testDbPath)) - { - try { File.Delete(_testDbPath); } catch { } - } - } - [Fact] public async Task ConcurrentUpsert_AllOperationsComplete() { diff --git a/src/tests/LiteDocumentStore.UnitTests/ExpressionToJsonPathTests.cs b/src/tests/LiteDocumentStore.UnitTests/ExpressionToJsonPathTests.cs deleted file mode 100644 index a0b2a9f..0000000 --- a/src/tests/LiteDocumentStore.UnitTests/ExpressionToJsonPathTests.cs +++ /dev/null @@ -1,296 +0,0 @@ -using System.Linq.Expressions; -using Xunit; - -namespace LiteDocumentStore.UnitTests; - -public class ExpressionToJsonPathTests -{ - private class TestModel - { - public string Name { get; set; } = ""; - public string Email { get; set; } = ""; - public int Age { get; set; } - public Address Address { get; set; } = new(); - public List Tags { get; set; } = []; - } - - private class Address - { - public string Street { get; set; } = ""; - public string City { get; set; } = ""; - public string ZipCode { get; set; } = ""; - } - - [Fact] - public void Translate_SimpleProperty_ReturnsCorrectJsonPath() - { - // Arrange - Expression> expr = x => x.Name; - - // Act - var result = ExpressionToJsonPath.Translate(expr); - - // Assert - Assert.Equal("$.Name", result); - } - - [Fact] - public void Translate_NestedProperty_ReturnsCorrectJsonPath() - { - // Arrange - Expression> expr = x => x.Address.City; - - // Act - var result = ExpressionToJsonPath.Translate(expr); - - // Assert - Assert.Equal("$.Address.City", result); - } - - [Fact] - public void Translate_DeeplyNestedProperty_ReturnsCorrectJsonPath() - { - // Arrange - Expression> expr = x => x.Address.ZipCode; - - // Act - var result = ExpressionToJsonPath.Translate(expr); - - // Assert - Assert.Equal("$.Address.ZipCode", result); - } - - [Fact] - public void Translate_ArrayIndexer_ReturnsCorrectJsonPath() - { - // Arrange - Expression> expr = x => x.Tags[0]; - - // Act - var result = ExpressionToJsonPath.Translate(expr); - - // Assert - Assert.Equal("$.Tags[0]", result); - } - - [Fact] - public void Translate_ArrayIndexerWithVariable_ReturnsCorrectJsonPath() - { - // Arrange - int index = 2; - Expression> expr = x => x.Tags[index]; - - // Act - var result = ExpressionToJsonPath.Translate(expr); - - // Assert - Assert.Equal("$.Tags[2]", result); - } - - [Fact] - public void Translate_NullExpression_ThrowsArgumentNullException() - { - // Arrange - Expression> expr = null!; - - // Act & Assert - Assert.Throws(() => ExpressionToJsonPath.Translate(expr)); - } - - [Fact] - public void TranslatePredicate_SimpleEquality_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Name == "John"; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Name') = @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal("John", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_Inequality_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Age != 30; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Age') != @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal(30, parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_GreaterThan_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Age > 18; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Age') > @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal(18, parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_LessThanOrEqual_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Age <= 65; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Age') <= @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal(65, parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_AndCondition_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Age > 18 && x.Name == "John"; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("(json_extract(data, '$.Age') > @p0 AND json_extract(data, '$.Name') = @p1)", whereClause); - Assert.Equal(2, parameters.Count); - Assert.Equal(18, parameters["p0"]); - Assert.Equal("John", parameters["p1"]); - } - - [Fact] - public void TranslatePredicate_OrCondition_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Name == "John" || x.Name == "Jane"; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("(json_extract(data, '$.Name') = @p0 OR json_extract(data, '$.Name') = @p1)", whereClause); - Assert.Equal(2, parameters.Count); - Assert.Equal("John", parameters["p0"]); - Assert.Equal("Jane", parameters["p1"]); - } - - [Fact] - public void TranslatePredicate_NestedProperty_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Address.City == "New York"; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Address.City') = @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal("New York", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_StringContains_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Email.Contains("@example.com"); - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Email') LIKE @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal("%@example.com%", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_StringStartsWith_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Name.StartsWith("J"); - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Name') LIKE @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal("J%", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_StringEndsWith_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => x.Email.EndsWith(".com"); - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Email') LIKE @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal("%.com", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_WithClosureVariable_ReturnsCorrectWhereClause() - { - // Arrange - string searchName = "Alice"; - Expression> predicate = x => x.Name == searchName; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Name') = @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal("Alice", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_ComplexCondition_ReturnsCorrectWhereClause() - { - // Arrange - Expression> predicate = x => - (x.Age > 18 && x.Age < 65) || x.Name == "Admin"; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("((json_extract(data, '$.Age') > @p0 AND json_extract(data, '$.Age') < @p1) OR json_extract(data, '$.Name') = @p2)", whereClause); - Assert.Equal(3, parameters.Count); - Assert.Equal(18, parameters["p0"]); - Assert.Equal(65, parameters["p1"]); - Assert.Equal("Admin", parameters["p2"]); - } - - [Fact] - public void TranslatePredicate_NullPredicate_ThrowsArgumentNullException() - { - // Arrange - Expression> predicate = null!; - - // Act & Assert - Assert.Throws(() => ExpressionToJsonPath.TranslatePredicate(predicate)); - } -} - diff --git a/src/tests/LiteDocumentStore.UnitTests/LiteDocumentStore.UnitTests.csproj b/src/tests/LiteDocumentStore.UnitTests/LiteDocumentStore.UnitTests.csproj index 75a8fbe..0c80596 100644 --- a/src/tests/LiteDocumentStore.UnitTests/LiteDocumentStore.UnitTests.csproj +++ b/src/tests/LiteDocumentStore.UnitTests/LiteDocumentStore.UnitTests.csproj @@ -9,11 +9,11 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/tests/LiteDocumentStore.UnitTests/VirtualColumnTests.cs b/src/tests/LiteDocumentStore.UnitTests/VirtualColumnTests.cs deleted file mode 100644 index 4ab2c17..0000000 --- a/src/tests/LiteDocumentStore.UnitTests/VirtualColumnTests.cs +++ /dev/null @@ -1,303 +0,0 @@ -using System.Linq.Expressions; -using Xunit; - -namespace LiteDocumentStore.UnitTests; - -/// -/// Unit tests for virtual column translation in ExpressionToJsonPath. -/// Tests verify that when virtual columns are provided, the SQL generator -/// uses column references instead of json_extract() for optimal index usage. -/// -public class VirtualColumnTranslationTests -{ - private class TestModel - { - public string Name { get; set; } = ""; - public string Email { get; set; } = ""; - public int Age { get; set; } - public Address Address { get; set; } = new(); - } - - private class Address - { - public string Street { get; set; } = ""; - public string City { get; set; } = ""; - } - - #region Equality Comparisons - - [Fact] - public void TranslatePredicate_WithVirtualColumn_UsesColumnReference() - { - // Arrange - Expression> predicate = x => x.Name == "John"; - var virtualColumns = new Dictionary - { - ["$.Name"] = new VirtualColumnInfo("$.Name", "name_vc", "TEXT") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("[name_vc] = @p0", whereClause); - Assert.True(parameters.ContainsKey("p0")); - Assert.Equal("John", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_WithVirtualColumn_NestedProperty_UsesColumnReference() - { - // Arrange - Expression> predicate = x => x.Address.City == "New York"; - var virtualColumns = new Dictionary - { - ["$.Address.City"] = new VirtualColumnInfo("$.Address.City", "city", "TEXT") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("[city] = @p0", whereClause); - Assert.Equal("New York", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_WithVirtualColumn_InequalityOperator_UsesColumnReference() - { - // Arrange - Expression> predicate = x => x.Age != 30; - var virtualColumns = new Dictionary - { - ["$.Age"] = new VirtualColumnInfo("$.Age", "age", "INTEGER") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("[age] != @p0", whereClause); - Assert.Equal(30, parameters["p0"]); - } - - #endregion - - #region Comparison Operators - - [Fact] - public void TranslatePredicate_WithVirtualColumn_GreaterThan_UsesColumnReference() - { - // Arrange - Expression> predicate = x => x.Age > 18; - var virtualColumns = new Dictionary - { - ["$.Age"] = new VirtualColumnInfo("$.Age", "age", "INTEGER") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("[age] > @p0", whereClause); - Assert.Equal(18, parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_WithVirtualColumn_LessThanOrEqual_UsesColumnReference() - { - // Arrange - Expression> predicate = x => x.Age <= 65; - var virtualColumns = new Dictionary - { - ["$.Age"] = new VirtualColumnInfo("$.Age", "age", "INTEGER") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("[age] <= @p0", whereClause); - Assert.Equal(65, parameters["p0"]); - } - - #endregion - - #region String Methods - - [Fact] - public void TranslatePredicate_WithVirtualColumn_StringContains_UsesColumnReference() - { - // Arrange - Expression> predicate = x => x.Email.Contains("@example"); - var virtualColumns = new Dictionary - { - ["$.Email"] = new VirtualColumnInfo("$.Email", "email", "TEXT") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("[email] LIKE @p0", whereClause); - Assert.Equal("%@example%", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_WithVirtualColumn_StringStartsWith_UsesColumnReference() - { - // Arrange - Expression> predicate = x => x.Name.StartsWith("Jo"); - var virtualColumns = new Dictionary - { - ["$.Name"] = new VirtualColumnInfo("$.Name", "name_vc", "TEXT") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("[name_vc] LIKE @p0", whereClause); - Assert.Equal("Jo%", parameters["p0"]); - } - - [Fact] - public void TranslatePredicate_WithVirtualColumn_StringEndsWith_UsesColumnReference() - { - // Arrange - Expression> predicate = x => x.Email.EndsWith(".com"); - var virtualColumns = new Dictionary - { - ["$.Email"] = new VirtualColumnInfo("$.Email", "email", "TEXT") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("[email] LIKE @p0", whereClause); - Assert.Equal("%.com", parameters["p0"]); - } - - #endregion - - #region Mixed Virtual and Non-Virtual - - [Fact] - public void TranslatePredicate_MixedVirtualAndNonVirtual_UsesCorrectReferences() - { - // Arrange - Expression> predicate = x => x.Name == "John" && x.Age > 18; - var virtualColumns = new Dictionary - { - // Only Name has a virtual column, Age does not - ["$.Name"] = new VirtualColumnInfo("$.Name", "name_vc", "TEXT") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - // Name uses virtual column, Age uses json_extract - Assert.Equal("([name_vc] = @p0 AND json_extract(data, '$.Age') > @p1)", whereClause); - Assert.Equal("John", parameters["p0"]); - Assert.Equal(18, parameters["p1"]); - } - - [Fact] - public void TranslatePredicate_OrCondition_WithVirtualColumns_UsesCorrectReferences() - { - // Arrange - Expression> predicate = x => x.Name == "John" || x.Email == "jane@example.com"; - var virtualColumns = new Dictionary - { - ["$.Name"] = new VirtualColumnInfo("$.Name", "name_vc", "TEXT"), - ["$.Email"] = new VirtualColumnInfo("$.Email", "email", "TEXT") - }; - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("([name_vc] = @p0 OR [email] = @p1)", whereClause); - Assert.Equal("John", parameters["p0"]); - Assert.Equal("jane@example.com", parameters["p1"]); - } - - #endregion - - #region Fallback Behavior - - [Fact] - public void TranslatePredicate_WithNullVirtualColumns_UsesJsonExtract() - { - // Arrange - Expression> predicate = x => x.Name == "John"; - - // Act - passing null explicitly - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, null); - - // Assert - Assert.Equal("json_extract(data, '$.Name') = @p0", whereClause); - } - - [Fact] - public void TranslatePredicate_WithEmptyVirtualColumns_UsesJsonExtract() - { - // Arrange - Expression> predicate = x => x.Name == "John"; - var virtualColumns = new Dictionary(); - - // Act - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate, virtualColumns); - - // Assert - Assert.Equal("json_extract(data, '$.Name') = @p0", whereClause); - } - - [Fact] - public void TranslatePredicate_WithoutVirtualColumnsParam_UsesJsonExtract() - { - // Arrange - Expression> predicate = x => x.Name == "John"; - - // Act - not passing virtual columns at all (default parameter) - var (whereClause, parameters) = ExpressionToJsonPath.TranslatePredicate(predicate); - - // Assert - Assert.Equal("json_extract(data, '$.Name') = @p0", whereClause); - } - - #endregion -} - -/// -/// Unit tests for VirtualColumnInfo record. -/// -public class VirtualColumnInfoTests -{ - [Fact] - public void VirtualColumnInfo_RecordEquality_Works() - { - // Arrange - var info1 = new VirtualColumnInfo("$.Name", "name_vc", "TEXT"); - var info2 = new VirtualColumnInfo("$.Name", "name_vc", "TEXT"); - var info3 = new VirtualColumnInfo("$.Email", "email", "TEXT"); - - // Assert - Assert.Equal(info1, info2); - Assert.NotEqual(info1, info3); - } - - [Fact] - public void VirtualColumnInfo_Properties_AreSet() - { - // Arrange & Act - var info = new VirtualColumnInfo("$.Address.City", "city", "TEXT"); - - // Assert - Assert.Equal("$.Address.City", info.JsonPath); - Assert.Equal("city", info.ColumnName); - Assert.Equal("TEXT", info.ColumnType); - } -}