Skip to content

Refactor core pipeline to TSymbol generics + offset-only source maps; add IDirectiveModel; move text-only APIs; refresh docs/tests #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dsisco11 merged 3 commits into from
Jan 10, 2026
Merged

Refactor core pipeline to TSymbol generics + offset-only source maps; add IDirectiveModel; move text-only APIs; refresh docs/tests #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
93996c2
refactor: overhauling systems to be generic
dsisco11 Jan 10, 2026
24dbc35
chore: removing legacy directive types
dsisco11 Jan 10, 2026
69cc0b6
docs: update readme
dsisco11 Jan 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 67 additions & 48 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,33 +25,50 @@ dotnet add package TinyPreprocessor

## Quick Start

TinyPreprocessor requires three components:
TinyPreprocessor is a small pipeline that:

1. Parses directives from each resource.
2. Uses `IDirectiveModel<TDirective>` to decide which directives represent dependencies (and where they are).
3. Resolves dependencies via `IResourceResolver<TSymbol>` (building a dependency graph).
4. Topologically orders resources (dependencies first).
5. Merges them via `IMergeStrategy<TSymbol, TDirective, TContext>` while building a source map and collecting diagnostics.

TinyPreprocessor requires four components:

1. **`IDirectiveParser<TDirective>`** – Parses directives from resource content
2. **`IResourceResolver`** – Resolves references to actual resources
3. **`IMergeStrategy<TContext>`** – Combines resources into final output
2. **`IDirectiveModel<TDirective>`** – Interprets directive locations and dependency references
3. **`IResourceResolver<TSymbol>`** – Resolves references to actual resources
4. **`IMergeStrategy<TSymbol, TDirective, TContext>`** – Combines resources into final output

### Example: Simple Include Directive
### Example: Minimal In-Memory Includes

```csharp
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using TinyPreprocessor;
using TinyPreprocessor.Core;
using TinyPreprocessor.Diagnostics;
using TinyPreprocessor.Merging;
using TinyPreprocessor.Text;

// 1. Define your directive type
public sealed record IncludeDirective(string Path, Range Location) : IIncludeDirective
// 1) Define your directive type.
public sealed record IncludeDirective(string Reference, Range Location);

// 2) Provide directive semantics to the pipeline.
public sealed class IncludeDirectiveModel : IDirectiveModel<IncludeDirective>
{
public string Reference => Path;
public Range GetLocation(IncludeDirective directive) => directive.Location;

public bool TryGetReference(IncludeDirective directive, out string reference)
{
reference = directive.Reference;
return true;
}
}

// 2. Implement directive parser
public sealed class IncludeParser : IDirectiveParser<IncludeDirective>
// 3) Implement a tiny directive parser for lines like: #include other.txt
public sealed class IncludeParser : IDirectiveParser<char, IncludeDirective>
{
public IEnumerable<IncludeDirective> Parse(ReadOnlyMemory<char> content, ResourceId resourceId)
{
Expand All @@ -71,48 +88,49 @@ public sealed class IncludeParser : IDirectiveParser<IncludeDirective>
}
}

// 3. Implement resource resolver
public sealed class FileResolver : IResourceResolver
// 4) Implement an in-memory resolver.
public sealed class InMemoryResolver : IResourceResolver<char>
{
private readonly string _basePath;
private readonly IReadOnlyDictionary<ResourceId, string> _files;

public FileResolver(string basePath) => _basePath = basePath;
public InMemoryResolver(IReadOnlyDictionary<ResourceId, string> files) => _files = files;

public ValueTask<ResourceResolutionResult> ResolveAsync(
public ValueTask<ResourceResolutionResult<char>> ResolveAsync(
string reference,
IResource? context,
IResource<char>? context,
CancellationToken ct)
{
var fullPath = Path.Combine(_basePath, reference);

if (!File.Exists(fullPath))
if (!_files.TryGetValue(new ResourceId(reference), out var content))
{
return ValueTask.FromResult(new ResourceResolutionResult(
return ValueTask.FromResult(new ResourceResolutionResult<char>(
null,
new ResolutionFailedDiagnostic(reference, $"File not found: {fullPath}")));
new ResolutionFailedDiagnostic(reference, $"Not found: {reference}")));
}

var content = File.ReadAllText(fullPath);
var resource = new Resource(reference, content.AsMemory());
return ValueTask.FromResult(new ResourceResolutionResult(resource, null));
var resource = new Resource<char>(reference, content.AsMemory());
return ValueTask.FromResult(new ResourceResolutionResult<char>(resource, null));
}
}

// 4. Use the preprocessor
var parser = new IncludeParser();
var resolver = new FileResolver(@"C:\MyProject\src");
var merger = new ConcatenatingMergeStrategy<object>();
// 5) Wire everything together.
var files = new Dictionary<ResourceId, string>
{
["main.txt"] = "#include a.txt\nMAIN\n",
["a.txt"] = "A\n#include b.txt\n",
["b.txt"] = "B\n"
};

var parser = new IncludeParser();
var directiveModel = new IncludeDirectiveModel();
var resolver = new InMemoryResolver(files);
var merger = new ConcatenatingMergeStrategy<IncludeDirective, object>();
var context = new object();

var preprocessor = new Preprocessor<IncludeDirective, object>(parser, resolver, merger);

var rootContent = File.ReadAllText(@"C:\MyProject\src\main.txt");
var root = new Resource("main.txt", rootContent.AsMemory());
var preprocessor = new Preprocessor<char, IncludeDirective, object>(parser, directiveModel, resolver, merger);
var root = new Resource<char>("main.txt", files["main.txt"].AsMemory());

var result = await preprocessor.ProcessAsync(root, context);

if (result.Success)
if (!result.Diagnostics.HasErrors)
{
Console.WriteLine(result.Content.ToString());
}
Expand Down Expand Up @@ -146,37 +164,38 @@ Query the source map to trace output positions back to original files:
```csharp
var result = await preprocessor.ProcessAsync(root, context);

// Find where line 50, column 10 in output came from
var location = result.SourceMap.Query(new SourcePosition(line: 49, column: 9));
// Find where generated offset 0 in output came from
var location = result.SourceMap.Query(generatedOffset: 0);

if (location is not null)
{
var (line, col) = location.OriginalPosition.ToOneBased();
Console.WriteLine($"Originated from {location.Resource.Path} at line {line}, column {col}");
Console.WriteLine($"Originated from {location.Resource.Path} at original offset {location.OriginalOffset}");
}

// For precise diagnostic spans, query a range.
// The range may map to multiple original resources (e.g., it crosses file boundaries).
var ranges = result.SourceMap.Query(new SourcePosition(line: 49, column: 9), length: 20);
var ranges = result.SourceMap.QueryRangeByLength(generatedStartOffset: 0, length: 20);

foreach (var range in ranges)
{
Console.WriteLine(
$"Generated [{range.GeneratedStart} - {range.GeneratedEnd}) -> {range.Resource.Path} [{range.OriginalStart} - {range.OriginalEnd})");
$"Generated [{range.GeneratedStartOffset} - {range.GeneratedEndOffset}) -> {range.Resource.Path} [{range.OriginalStartOffset} - {range.OriginalEndOffset})");
}
```

## Custom Merge Strategy

Implement `IMergeStrategy<TContext>` for custom output formatting:
Implement `IMergeStrategy<TSymbol, TDirective, TContext>` for custom output formatting:

```csharp
public sealed class JsonMergeStrategy : IMergeStrategy<JsonMergeOptions>
public sealed record JsonMergeOptions;

public sealed class JsonMergeStrategy : IMergeStrategy<char, IncludeDirective, JsonMergeOptions>
{
public ReadOnlyMemory<char> Merge(
IReadOnlyList<ResolvedResource> orderedResources,
JsonMergeOptions context,
MergeContext mergeContext)
IReadOnlyList<ResolvedResource<char, IncludeDirective>> orderedResources,
JsonMergeOptions userContext,
MergeContext<char, IncludeDirective> mergeContext)
{
// Custom merge logic here
// Use mergeContext.SourceMapBuilder to record mappings.
Expand Down Expand Up @@ -204,8 +223,8 @@ public sealed class JsonMergeStrategy : IMergeStrategy<JsonMergeOptions>
┌─────────────────────────────────────────────────────────────┐
│ Preprocessor │
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐ │
│ │ IDirective │ │ IResource │ │ IMergeStrategy │ │
│ │ Parser │ │ Resolver │ │ │ │
│ │ Directive │ │ IResource<T> │ │ IMergeStrategy │ │
│ │ Parser/Model │ │ Resolver │ │ │ │
│ └──────────────┘ └──────────────┘ └──────────────────┘ │
│ │ │ │ │
│ ▼ ▼ ▼ │
Expand Down
28 changes: 14 additions & 14 deletions TinyPreprocessor.Tests/Core/ResourceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
namespace TinyPreprocessor.Tests.Core;

/// <summary>
/// Unit tests for <see cref="Resource"/>.
/// Unit tests for <see cref="Resource{TSymbol}"/>.
/// </summary>
public sealed class ResourceTests
{
Expand All @@ -17,7 +17,7 @@ public void Constructor_WithAllParameters_SetsPropertiesCorrectly()
var content = "Hello, World!".AsMemory();
var metadata = new Dictionary<string, object> { ["key"] = "value" };

var resource = new Resource(id, content, metadata);
var resource = new Resource<char>(id, content, metadata);

Assert.Equal(id, resource.Id);
Assert.Equal("Hello, World!", resource.Content.ToString());
Expand All @@ -31,7 +31,7 @@ public void Constructor_WithoutMetadata_SetsMetadataToNull()
var id = new ResourceId("test.txt");
var content = "Content".AsMemory();

var resource = new Resource(id, content);
var resource = new Resource<char>(id, content);

Assert.Null(resource.Metadata);
}
Expand All @@ -42,7 +42,7 @@ public void Constructor_WithEmptyContent_Succeeds()
var id = new ResourceId("empty.txt");
var content = ReadOnlyMemory<char>.Empty;

var resource = new Resource(id, content);
var resource = new Resource<char>(id, content);

Assert.True(resource.Content.IsEmpty);
}
Expand All @@ -55,9 +55,9 @@ public void Constructor_WithEmptyContent_Succeeds()
public void Id_ReturnsResourceId()
{
ResourceId id = "resource/path.txt";
var resource = new Resource(id, "content".AsMemory());
var resource = new Resource<char>(id, "content".AsMemory());

IResource iResource = resource;
IResource<char> iResource = resource;

Assert.Equal(id, iResource.Id);
}
Expand All @@ -66,7 +66,7 @@ public void Id_ReturnsResourceId()
public void Content_ReturnsReadOnlyMemory()
{
const string contentString = "Test content with special chars: éàü";
var resource = new Resource("test.txt", contentString.AsMemory());
var resource = new Resource<char>("test.txt", contentString.AsMemory());

Assert.Equal(contentString, resource.Content.ToString());
}
Expand All @@ -79,8 +79,8 @@ public void Content_ReturnsReadOnlyMemory()
public void Equals_SameIdAndContent_ReturnsTrue()
{
var content = "Same content".AsMemory();
var resource1 = new Resource("file.txt", content);
var resource2 = new Resource("file.txt", content);
var resource1 = new Resource<char>("file.txt", content);
var resource2 = new Resource<char>("file.txt", content);

// Records compare by value for properties
Assert.Equal(resource1.Id, resource2.Id);
Expand All @@ -90,16 +90,16 @@ public void Equals_SameIdAndContent_ReturnsTrue()
public void Equals_DifferentId_ReturnsFalse()
{
var content = "Content".AsMemory();
var resource1 = new Resource("file1.txt", content);
var resource2 = new Resource("file2.txt", content);
var resource1 = new Resource<char>("file1.txt", content);
var resource2 = new Resource<char>("file2.txt", content);

Assert.NotEqual(resource1.Id, resource2.Id);
}

[Fact]
public void With_CreatesModifiedCopy()
{
var original = new Resource("original.txt", "original content".AsMemory());
var original = new Resource<char>("original.txt", "original content".AsMemory());
var newContent = "new content".AsMemory();

var modified = original with { Content = newContent };
Expand All @@ -123,7 +123,7 @@ public void Metadata_WhenProvided_IsAccessible()
["IsGenerated"] = true
};

var resource = new Resource("test.txt", "content".AsMemory(), metadata);
var resource = new Resource<char>("test.txt", "content".AsMemory(), metadata);

Assert.Equal("Test Author", resource.Metadata!["Author"]);
Assert.Equal(42, resource.Metadata["LineCount"]);
Expand All @@ -134,7 +134,7 @@ public void Metadata_WhenProvided_IsAccessible()
public void Metadata_IsReadOnly()
{
var metadata = new Dictionary<string, object> { ["key"] = "value" };
var resource = new Resource("test.txt", "content".AsMemory(), metadata);
var resource = new Resource<char>("test.txt", "content".AsMemory(), metadata);

// IReadOnlyDictionary doesn't have Add method
Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(resource.Metadata);
Expand Down
Loading
Loading