diff --git a/include/tokenizer.h b/include/tokenizer.h index 7338819..cacbc3a 100644 --- a/include/tokenizer.h +++ b/include/tokenizer.h @@ -1,8 +1,60 @@ // @Eamon2009 +/*// ------------------------------------------------------- +// overview and some NOTES +//----------------------------------------------------------- +// +// This header file has a dual-mode Byte-Pair Encoding +// (BPE) tokenizer and dataset loader (DataLoader) for repo llm.cpp +// train pipeline. It supports raw text corpus training and +// memory-mapped (mmap) binary shards also training. + +// 1. text Mode: Reads raw text files, trains a BPE tokenizer from scratch up to a +// target vocabulary size, caches vocabularies to disk (.tokenizer.bin), and +// splits tokens into contiguous train/validation memory vectors. +// 2. SHARDED Mode: Bypasses heavy RAM overhead by using memory-mapping +// (POSIX mmap / Win32 File Mapping) to stream pre-tokenized binary shards (.bin). +// Shards utilize a fixed 1KB header followed by a flat stream of uint16_t +// token IDs, capping vocabulary support at 65,536 entries. +// notes: +// * Platform-Specific Memory Mapping: look `MMapShard::open()` for platform +// branches. Windows uses CreateFileMappingA/MapViewOfFile, while POSIX systems +// rely on open(), mmap(), and madvise(MADV_RANDOM) to optimize non-sequential +// batch sampling hints for kerenels +// * Move-Only Semantics: `MMapShard` implements explicit move-only semantics +// (`= delete` on copy constructors/operators) to safely manage file descriptors +// and resource handles without double-free errors +// * Cache-Aligned Index Structures: `BPEIndex` is explicitly alignas(16) to ensure +// optimal L1/L2 cache line utilization during fast doubly-linked list traversal +// in BPE merging routine +// * Packed Hash Keys: Pair keys for merge ranks are packed into a single uint64_t +// using bit shifts (`((uint64_t)left << 32) | (uint32_t)right`) to optimize +// hash map lookups (`std::unordered_map`) +// * Parallelized Batch Sampling: OpenMP (`#pragma omp parallel for`) is leveraged +// across batch sampling (`get_batch_text` and `get_batch_sharded`) and base encoding +// to parallelize random number generation and index lookups safely using thread-local +// random engines (std::mt19937) + +Shard Capacity Limit: Because shards serialize tokens as `uint16_t`, +vocabularies exceeding 65,536 entries will cause a hard runtime exception +(`write_shards` check). Do not alter token datatype widths without redesigning +the header structure (`SHARD_HEADER_INTS`). + +Boundary Conditions in Sharded Sampling: `get_batch_sharded` contains both a +fast path (contiguous memory inside a single shard) and a rare path (blocks +crossing shard boundaries via prefix-sum lookups using `std::upper_bound`). +Changes to indexing logic must maintain safety against off-by-one segment faults. + +Mutability and Thread Safety: While data streaming via mmap is read-only and +inherently thread-safe across multiple worker threads, ensure that external +callers do not concurrently mutate vocabulary mappings (`token_to_id`, `vocab`) +while inference/encoding or training loops are executing. +// ----------------------------------------==--------------------------------------*/ #pragma once #include "config/config.h" #include +#include +#include #include #include #include @@ -15,8 +67,44 @@ #include #include +// Directory scanning / mkdir via std::filesystem -- portable across +// Linux/macOS/Windows, no platform #ifdef needed for this part. +#include + +// Memory-mapping is platform-specific: POSIX mmap vs Win32 file mappings. +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +// #define NOMINMAX +#include +#else +#include +#include +#include +#include +#endif + +namespace fs = std::filesystem; + +// Shard binary format : fixed 1KB header followed by a flat +// stream of uint16_t token ids. uint16_t caps vocab at 65536 entries, which +// covers essentially every BPE config people run (32k/50k/64k) at half the +// footprint of int32 -- shards are twice as fast to page in and to scan. +static constexpr int32_t SHARD_MAGIC = 20240801; +static constexpr int32_t SHARD_VERSION = 1; +static constexpr size_t SHARD_HEADER_INTS = 256; // 1024 bytes, room to grow +static constexpr size_t SHARD_HEADER_BYTES = SHARD_HEADER_INTS * sizeof(int32_t); +#ifndef SHARD_SIZE_TOKENS +#define SHARD_SIZE_TOKENS (100ull * 1000ull * 1000ull) // 100M tokens/shard (~200MB) +#endif + struct DataLoader { + enum class DataMode + { + TEXT, // raw .txt -> BPE-trained, fully in RAM (small/medium datasets) + SHARDED // directory of pre-tokenized .bin shards, mmap-streamed + }; + std::vector vocab; std::unordered_map token_to_id; std::vector> merges; @@ -27,8 +115,10 @@ struct DataLoader int base_vocab_size{0}; int vocab_size{0}; - std::vector train_data; - std::vector val_data; + DataMode mode{DataMode::TEXT}; + + std::vector train_data; // populated only in TEXT mode + std::vector val_data; // populated only in TEXT mode // 16-byte aligned struct for optimal L1/L2 cache line packing struct alignas(16) BPEIndex @@ -43,11 +133,223 @@ struct DataLoader { return ((uint64_t)(uint32_t)left << 32) | (uint32_t)right; } + // mmap'd shard: one .bin file, read-only, paged in lazily by the OS. + // Move-only (owns an fd + mapping). + + struct MMapShard + { +#ifdef _WIN32 + HANDLE hFile{INVALID_HANDLE_VALUE}; + HANDLE hMap{nullptr}; +#else + int fd{-1}; +#endif + void *map_base{nullptr}; + size_t map_size{0}; + const uint16_t *data{nullptr}; + uint64_t num_tokens{0}; + std::string path; + + MMapShard() = default; + MMapShard(const MMapShard &) = delete; + MMapShard &operator=(const MMapShard &) = delete; + + MMapShard(MMapShard &&o) noexcept + { + *this = std::move(o); + } + MMapShard &operator=(MMapShard &&o) noexcept + { + if (this != &o) + { + release(); +#ifdef _WIN32 + hFile = o.hFile; + hMap = o.hMap; + o.hFile = INVALID_HANDLE_VALUE; + o.hMap = nullptr; +#else + fd = o.fd; + o.fd = -1; +#endif + map_base = o.map_base; + map_size = o.map_size; + data = o.data; + num_tokens = o.num_tokens; + path = std::move(o.path); + o.map_base = nullptr; + o.map_size = 0; + o.data = nullptr; + o.num_tokens = 0; + } + return *this; + } + + ~MMapShard() + { + release(); + } + + void release() + { +#ifdef _WIN32 + if (map_base) + UnmapViewOfFile(map_base); + if (hMap) + CloseHandle(hMap); + if (hFile != INVALID_HANDLE_VALUE) + CloseHandle(hFile); + hMap = nullptr; + hFile = INVALID_HANDLE_VALUE; +#else + if (map_base && map_base != MAP_FAILED) + munmap(map_base, map_size); + if (fd >= 0) + ::close(fd); + fd = -1; +#endif + map_base = nullptr; + } + + void open(const std::string &p) + { + path = p; + +#ifdef _WIN32 + hFile = CreateFileA(p.c_str(), + GENERIC_READ, + FILE_SHARE_READ, + nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (hFile == INVALID_HANDLE_VALUE) + throw std::runtime_error("[Shard] Cannot open: " + p); + + LARGE_INTEGER size{}; + if (!GetFileSizeEx(hFile, &size)) + throw std::runtime_error("[Shard] GetFileSizeEx failed: " + p); + map_size = (size_t)size.QuadPart; + if (map_size <= SHARD_HEADER_BYTES) + throw std::runtime_error("[Shard] File too small to be a valid shard: " + + p); + + hMap = CreateFileMappingA(hFile, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (!hMap) + throw std::runtime_error("[Shard] CreateFileMapping failed: " + p); + + map_base = MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0); + if (!map_base) + throw std::runtime_error("[Shard] MapViewOfFile failed: " + p); + // (No direct equivalent of madvise(MADV_RANDOM) on Windows; + // PrefetchVirtualMemory exists on Win8+ but is opt-in/sequential- + // oriented, so we simply skip it here -- random access still + // works correctly, just without the readahead hint.) +#else + fd = ::open(p.c_str(), O_RDONLY); + if (fd < 0) + throw std::runtime_error("[Shard] Cannot open: " + p); + + struct stat st{}; + if (fstat(fd, &st) != 0) + throw std::runtime_error("[Shard] fstat failed: " + p); + map_size = (size_t)st.st_size; + if (map_size <= SHARD_HEADER_BYTES) + throw std::runtime_error("[Shard] File too small to be a valid shard: " + + p); + + map_base = mmap(nullptr, map_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (map_base == MAP_FAILED) + throw std::runtime_error("[Shard] mmap failed: " + p); + + // Batches sample random offsets, not sequential scans -- tell + // the kernel not to bother with aggressive readahead. + madvise(map_base, map_size, MADV_RANDOM); +#endif + + const int32_t *header = reinterpret_cast(map_base); + if (header[0] != SHARD_MAGIC) + throw std::runtime_error("[Shard] Bad magic (not a shard file): " + p); + if (header[1] != SHARD_VERSION) + throw std::runtime_error("[Shard] Unsupported shard version: " + p); + + uint64_t lo = (uint32_t)header[3]; + uint64_t hi = (uint32_t)header[4]; + num_tokens = (hi << 32) | lo; + + data = reinterpret_cast( + reinterpret_cast(map_base) + SHARD_HEADER_BYTES); + + uint64_t expected_bytes = SHARD_HEADER_BYTES + num_tokens * sizeof(uint16_t); + if (expected_bytes != map_size) + throw std::runtime_error("[Shard] Size/header mismatch in: " + p); + } + }; + + // A split (train/val) as a virtual concatenation of shards, so batch + // sampling can treat many small files as one big logical stream. + struct ShardedSplit + { + std::vector shards; + std::vector prefix; // prefix[i] = tokens before shard i + uint64_t total_tokens{0}; + + void add(const std::string &path) + { + MMapShard s; + s.open(path); + total_tokens += s.num_tokens; + shards.push_back(std::move(s)); + } + + void build_prefix() + { + prefix.assign(shards.size() + 1, 0); + for (size_t i = 0; i < shards.size(); ++i) + prefix[i + 1] = prefix[i] + shards[i].num_tokens; + } + + inline size_t locate(uint64_t global_idx) const + { + auto it = std::upper_bound(prefix.begin(), prefix.end(), global_idx); + return (size_t)(it - prefix.begin()) - 1; + } + + inline int token_at(uint64_t global_idx) const + { + size_t s = locate(global_idx); + uint64_t local = global_idx - prefix[s]; + return (int)shards[s].data[local]; + } + }; + + ShardedSplit shard_train; + ShardedSplit shard_val; + // Public entry point. Auto-detects at runtime: + // - path is a directory -> shard mode (llm.exe data/shards) + // - path is a .txt file -> classic BPE (llm.exe data/input.txt) void load(const std::string &path, int target_vocab = BPE_VOCAB_SIZE, double train_split = TRAIN_SPLIT) { + if (is_directory(path)) + { + load_shards(path); + } + else + { + load_text(path, target_vocab, train_split); + } + } + // TEXT mode: unchanged behavior, plus a vocab cache so re-running on + // the same file doesn't re-train BPE from scratch every time. + void load_text(const std::string &path, + int target_vocab = BPE_VOCAB_SIZE, + double train_split = TRAIN_SPLIT) + { + mode = DataMode::TEXT; + std::ifstream f(path); if (!f.is_open()) throw std::runtime_error("[DataLoader] Cannot open file: " + path); @@ -62,7 +364,21 @@ struct DataLoader std::cout << "[BPE] Target vocab size: " << target_vocab << "\n"; std::cout.flush(); - std::vector data = train_bpe(text, target_vocab); + std::vector data; + std::string cache_path = path + ".tokenizer.bin"; + if (load_vocab_if_matches(cache_path, target_vocab)) + { + std::cout << "[BPE] Loaded cached vocab from " << cache_path + << " (skipping BPE training)\n"; + std::cout.flush(); + data = apply_merges(base_encode(text)); + } + else + { + data = train_bpe(text, target_vocab); + save_vocab(cache_path); + std::cout << "[BPE] Cached vocab to " << cache_path << "\n"; + } int n = (int)(train_split * (double)data.size()); train_data = std::vector(data.begin(), data.begin() + n); @@ -76,6 +392,59 @@ struct DataLoader std::cout << "[DATA] Train tokens : " << train_data.size() << "\n"; std::cout << "[DATA] Val tokens : " << val_data.size() << "\n"; } + // SHARD mode: mmap every train_*.bin / val_*.bin in the directory. + // Nothing is copied into RAM up front -- pages fault in on demand + // and the OS evicts them under memory pressure, so this scales to + // corpora much larger than available RAM. + void load_shards(const std::string &dir) + { + mode = DataMode::SHARDED; + + std::vector train_files = list_files_matching(dir, "train_", ".bin"); + std::vector val_files = list_files_matching(dir, "val_", ".bin"); + + if (train_files.empty()) + throw std::runtime_error("[DataLoader] No train_*.bin shards found in: " + dir); + if (val_files.empty()) + throw std::runtime_error("[DataLoader] No val_*.bin shards found in: " + dir); + + std::sort(train_files.begin(), train_files.end()); + std::sort(val_files.begin(), val_files.end()); + + for (auto &p : train_files) + shard_train.add(p); + for (auto &p : val_files) + shard_val.add(p); + + shard_train.build_prefix(); + shard_val.build_prefix(); + + std::string vocab_path = dir + "/tokenizer.bin"; + std::ifstream check(vocab_path, std::ios::binary); + if (check.good()) + { + check.close(); + load_vocab(vocab_path); + std::cout << "[SHARD] Loaded tokenizer vocab from " << vocab_path << "\n"; + } + else + { + std::cout << "[SHARD] Warning: no tokenizer.bin found in " << dir + << " -- encode()/decode() unavailable, training ids only.\n"; + } + + std::cout << "[SHARD] Train shards : " << shard_train.shards.size() << " (" + << shard_train.total_tokens << " tokens)\n"; + std::cout << "[SHARD] Val shards : " << shard_val.shards.size() << " (" + << shard_val.total_tokens << " tokens)\n"; + std::cout.flush(); + + if (shard_train.total_tokens <= (uint64_t)BLOCK_SIZE || + shard_val.total_tokens <= (uint64_t)BLOCK_SIZE) + throw std::runtime_error( + "[DataLoader] Sharded dataset too small for BLOCK_SIZE=" + + std::to_string(BLOCK_SIZE)); + } std::vector encode(const std::string &text) const { @@ -91,9 +460,221 @@ struct DataLoader out += vocab[id]; return out; } - + // Unified batch sampler -- identical signature/behavior for callers + // regardless of whether we're backed by RAM vectors or mmap shards. std::pair, std::vector> get_batch(const std::string &split, int batch_size, int block_size, std::mt19937 &rng) const + { + if (mode == DataMode::SHARDED) + return get_batch_sharded(split, batch_size, block_size, rng); + return get_batch_text(split, batch_size, block_size, rng); + } + // Shard-writer: turn an already-loaded corpus (TEXT mode) into a + // directory of .bin shards using the *same* trained vocab, so you + // can pretokenize once and reuse across many training runs. + // loader.load_text("data/input.txt"); + // loader.write_shards("data/shards"); + void write_shards(const std::string &out_dir, + uint64_t shard_size_tokens = SHARD_SIZE_TOKENS) const + { + if (vocab.empty()) + throw std::runtime_error( + "[SHARD] No vocab trained/loaded -- call load_text() first"); + if (vocab.size() > 65536) + throw std::runtime_error("[SHARD] vocab_size " + std::to_string(vocab.size()) + + " exceeds uint16_t shard capacity (65536)"); + if (train_data.empty() || val_data.empty()) + throw std::runtime_error( + "[SHARD] No train/val data in memory -- call load_text() first"); + + make_directory(out_dir); + write_split_shards(out_dir, "train", train_data, shard_size_tokens); + write_split_shards(out_dir, "val", val_data, shard_size_tokens); + save_vocab(out_dir + "/tokenizer.bin"); + + std::cout << "[SHARD] Wrote shards + tokenizer.bin to " << out_dir << "\n"; + } + + // Vocab persistence (binary): lets shard mode decode(), and lets + // TEXT mode skip re-training BPE on repeat runs of the same file. + + void save_vocab(const std::string &path) const + { + std::ofstream f(path, std::ios::binary); + if (!f.is_open()) + throw std::runtime_error("[Vocab] Cannot write: " + path); + + uint32_t magic = 0x544B5631; // 'TKV1' + write_u32(f, magic); + write_u32(f, (uint32_t)base_vocab_size); + write_u32(f, (uint32_t)vocab_size); + write_u32(f, (uint32_t)vocab.size()); + for (const auto &tok : vocab) + { + write_u32(f, (uint32_t)tok.size()); + f.write(tok.data(), (std::streamsize)tok.size()); + } + write_u32(f, (uint32_t)merges.size()); + for (const auto &m : merges) + { + write_u32(f, (uint32_t)m.first); + write_u32(f, (uint32_t)m.second); + } + } + + void load_vocab(const std::string &path) + { + std::ifstream f(path, std::ios::binary); + if (!f.is_open()) + throw std::runtime_error("[Vocab] Cannot read: " + path); + + uint32_t magic = read_u32(f); + if (magic != 0x544B5631) + throw std::runtime_error("[Vocab] Bad magic (not a tokenizer.bin): " + path); + + base_vocab_size = (int)read_u32(f); + vocab_size = (int)read_u32(f); + + uint32_t n_vocab = read_u32(f); + vocab.clear(); + token_to_id.clear(); + vocab.reserve(n_vocab); + for (uint32_t i = 0; i < n_vocab; ++i) + { + uint32_t len = read_u32(f); + std::string tok(len, '\0'); + f.read(&tok[0], (std::streamsize)len); + token_to_id[tok] = (int)vocab.size(); + vocab.push_back(std::move(tok)); + } + + uint32_t n_merges = read_u32(f); + merges.clear(); + merge_rank.clear(); + merges.reserve(n_merges); + for (uint32_t i = 0; i < n_merges; ++i) + { + int left = (int)read_u32(f); + int right = (int)read_u32(f); + merges.push_back({left, right}); + merge_rank[make_pair_key(left, right)] = (int)i; + } + } + + private: + // small binary I/O helpers - + static void write_u32(std::ofstream &f, uint32_t v) + { + f.write(reinterpret_cast(&v), sizeof(v)); + } + static uint32_t read_u32(std::ifstream &f) + { + uint32_t v = 0; + f.read(reinterpret_cast(&v), sizeof(v)); + return v; + } + + bool load_vocab_if_matches(const std::string &cache_path, int target_vocab) + { + std::ifstream check(cache_path, std::ios::binary); + if (!check.good()) + return false; + check.close(); + try + { + load_vocab(cache_path); + } + catch (const std::exception &) + { + return false; + } + return vocab_size == target_vocab; + } + + // filesystem helpers (std::filesystem portable) + static bool is_directory(const std::string &path) + { + std::error_code ec; + return fs::is_directory(path, ec); // false (not throw) if path doesn't exist + } + + static void make_directory(const std::string &path) + { + std::error_code ec; + fs::create_directories(path, ec); + if (ec && !fs::is_directory(path)) + throw std::runtime_error("[SHARD] Cannot create directory: " + path + " (" + + ec.message() + ")"); + } + + static std::vector list_files_matching(const std::string &dir, + const std::string &prefix, + const std::string &suffix) + { + std::vector out; + std::error_code ec; + fs::directory_iterator it(dir, ec); + if (ec) + throw std::runtime_error("[DataLoader] Cannot open directory: " + dir); + + for (const auto &entry : it) + { + if (!entry.is_regular_file()) + continue; + std::string name = entry.path().filename().string(); + bool has_prefix = name.compare(0, prefix.size(), prefix) == 0; + bool has_suffix = + name.size() >= suffix.size() && + name.compare(name.size() - suffix.size(), suffix.size(), suffix) == 0; + if (has_prefix && has_suffix) + out.push_back(entry.path().string()); + } + return out; + } + + void write_split_shards(const std::string &out_dir, + const std::string &split_name, + const std::vector &ids, + uint64_t shard_size_tokens) const + { + uint64_t total = ids.size(); + uint64_t shard_idx = 0; + for (uint64_t offset = 0; offset < total; offset += shard_size_tokens, ++shard_idx) + { + uint64_t count = std::min(shard_size_tokens, total - offset); + + char name_buf[64]; + std::snprintf(name_buf, + sizeof(name_buf), + "%s_%06llu.bin", + split_name.c_str(), + (unsigned long long)shard_idx); + std::string path = out_dir + "/" + name_buf; + + std::ofstream f(path, std::ios::binary); + if (!f.is_open()) + throw std::runtime_error("[SHARD] Cannot write: " + path); + + int32_t header[SHARD_HEADER_INTS] = {0}; + header[0] = SHARD_MAGIC; + header[1] = SHARD_VERSION; + header[2] = 0; // dtype: 0 = uint16 + header[3] = (int32_t)(uint32_t)(count & 0xFFFFFFFFull); + header[4] = (int32_t)(uint32_t)(count >> 32); + f.write(reinterpret_cast(header), sizeof(header)); + + std::vector buf(count); + for (uint64_t i = 0; i < count; ++i) + buf[i] = (uint16_t)ids[offset + i]; + f.write(reinterpret_cast(buf.data()), + (std::streamsize)(count * sizeof(uint16_t))); + } + } + + std::pair, std::vector> get_batch_text(const std::string &split, + int batch_size, + int block_size, + std::mt19937 &rng) const { const std::vector &d = (split == "train") ? train_data : val_data; std::uniform_int_distribution dist(0, (int)d.size() - block_size - 1); @@ -101,7 +682,6 @@ struct DataLoader std::vector x(batch_size * block_size); std::vector y(batch_size * block_size); - // Generate deterministic seeds for parallel threads std::vector seeds(batch_size); for (int b = 0; b < batch_size; ++b) seeds[b] = rng(); @@ -120,12 +700,62 @@ struct DataLoader return {x, y}; } - private: + // ---- SHARD-mode batch sampling ---- + // Samples a global offset over the virtual concatenation of shards. + // Fast path: the whole block lives inside one shard -> direct mmap + // pointer reads. Rare path (block straddles a shard boundary): + // falls back to a per-token lookup via the prefix-sum index. + std::pair, std::vector> get_batch_sharded(const std::string &split, + int batch_size, + int block_size, + std::mt19937 &rng) const + { + const ShardedSplit &s = (split == "train") ? shard_train : shard_val; + std::uniform_int_distribution dist(0, s.total_tokens - block_size - 1); + + std::vector x(batch_size * block_size); + std::vector y(batch_size * block_size); + + std::vector seeds(batch_size); + for (int b = 0; b < batch_size; ++b) + seeds[b] = rng(); + +#pragma omp parallel for schedule(static) + for (int b = 0; b < batch_size; ++b) + { + std::mt19937 thread_rng(seeds[b]); + uint64_t start = dist(thread_rng); + + size_t shard_i = s.locate(start); + uint64_t local = start - s.prefix[shard_i]; + const MMapShard &shard = s.shards[shard_i]; + + if (local + (uint64_t)block_size + 1 <= shard.num_tokens) + { + // Fast path: contiguous within one shard. + for (int t = 0; t < block_size; ++t) + { + x[b * block_size + t] = (int)shard.data[local + t]; + y[b * block_size + t] = (int)shard.data[local + t + 1]; + } + } + else + { + // Rare path: crosses a shard boundary. + for (int t = 0; t < block_size; ++t) + { + x[b * block_size + t] = s.token_at(start + t); + y[b * block_size + t] = s.token_at(start + t + 1); + } + } + } + return {x, y}; + } + std::vector base_encode(const std::string &text) const { std::vector ids(text.size()); - // Parallel character mapping for large text buffers if (text.size() > 10000) { #pragma omp parallel for schedule(static) @@ -161,7 +791,6 @@ struct DataLoader nodes[i] = {ids[i], i - 1, i + 1, 1}; nodes[n - 1].next = -1; - // Priority queue to merge pairs in lowest rank order first (O(N log N)) using PQItem = std::pair; // {rank, left_index} std::priority_queue, std::greater> pq; @@ -251,7 +880,6 @@ struct DataLoader if (n > 0) list[n - 1].next = -1; - // Inverted index mapping pair key -> list of indices where it occurs std::unordered_map> pair_pos; pair_pos.reserve(n); @@ -271,7 +899,6 @@ struct DataLoader uint64_t best_key = 0; size_t max_count = 0; - // Scan unique key pairs to find max frequency for (auto const &[key, pos_vec] : pair_pos) { if (pos_vec.size() > max_count) @@ -296,7 +923,6 @@ struct DataLoader auto occs = std::move(pair_pos[best_key]); pair_pos.erase(best_key); - // Update only actual locations where the pair appears for (int head : occs) { if (!list[head].active || list[head].id != left) @@ -351,4 +977,4 @@ struct DataLoader std::cout << "[BPE] Done. Final vocab size: " << vocab_size << "\n"; return final_ids; } -}; \ No newline at end of file +};