diff --git a/.gitignore b/.gitignore index dbb3303e3..f96838d01 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,6 @@ docs/docs/public/c-reference # Clangd config compile_flags.txt + +# Go +!/go/herb/go.mod diff --git a/Rakefile b/Rakefile index 4f78fec4d..19c41f77b 100644 --- a/Rakefile +++ b/Rakefile @@ -151,7 +151,8 @@ namespace :prism do "Rakefile", "src/", "include/", - "templates/" + "templates/", + "build/" ] files.each do |file| diff --git a/go/Makefile b/go/Makefile new file mode 100644 index 000000000..776e8f54b --- /dev/null +++ b/go/Makefile @@ -0,0 +1,14 @@ +# generates go bindings +build: + c-for-go herb.yml + +# cleanup generated files +clean: + rm -f herb/cgo_helpers.go herb/cgo_helpers.h herb/cgo_helpers.c + rm -f herb/const.go herb/doc.go herb/types.go + rm -f herb/herb.go + rm -rf bin/ + +# test build and run tests +test: + cd herb && go build && go test diff --git a/go/README.md b/go/README.md new file mode 100644 index 000000000..891fde900 --- /dev/null +++ b/go/README.md @@ -0,0 +1,67 @@ +# Herb Go Bindings + +Go bindings for Herb - Powerful and seamless HTML-aware ERB parsing and tooling. + +## Building + +### Prerequisites + +- Go 1.20 or later +- C compiler (gcc/clang) +- Herb C library built from parent directory +- [c-for-go](https://github.com/xlab/c-for-go) go-lang bindings generator + +### Build + +```bash +# Build herb C library from project root +cd .. +make build/libherb.a +make prism + +# Generate bindings +cd go +make build +``` + +## Usage + +### As a Library + +```go +package main + +import ( + "fmt" + "github.com/marcoroth/herb" +) + +func main() { + template := "

<%= title %>

" + + // Lex ERB template + tokens := herb.Lex(template) + if tokens != nil { + fmt.Println("Lexing successful") + } + + // Parse ERB template + ast := herb.Parse(template, nil) + if ast != nil { + fmt.Println("Parsing successful") + } + + // Extract Ruby code + ruby := herb.Extract(template, 0) + if ruby != nil { + fmt.Printf("Ruby: %s\n", string(*ruby)) + } +} +``` + +## Testing + +```bash +cd herb +go test -v +``` diff --git a/go/herb.yml b/go/herb.yml new file mode 100644 index 000000000..1a09fe3ce --- /dev/null +++ b/go/herb.yml @@ -0,0 +1,59 @@ +--- +GENERATOR: + PackageName: herb + PackageDescription: "Go bindings for Herb - HTML-aware ERB parser" + PackageLicense: "MIT" + Includes: + - herb_go.h + Options: + SafeStrings: true + FlagGroups: + - {name: CFLAGS, flags: ["-I..", "-I../src/include", "-I../src"]} + - {name: LDFLAGS, flags: ["${SRCDIR}/../../build/libherb.a", "${SRCDIR}/../../vendor/prism/build/libprism.a"]} + +PARSER: + IncludePaths: + - "." + SourcesPaths: + - ./herb/herb_go.h + +TRANSLATOR: + Rules: + global: + - { action: accept, from: "^herb_" } + - { action: accept, from: "^hb_" } + - { action: accept, from: "^AST_" } + - { action: accept, from: "^parser_" } + + function: + - { action: accept, from: "^herb_" } + - { action: replace, from: "^herb_", to: "" } + - { load: snakecase, transform: lower } + - { transform: export } + + type: + - { action: accept, from: "_[tT]$" } + - { action: replace, from: "_[tT]$", to: "" } + - { load: snakecase, transform: pascalcase } + + MemTips: + - { target: "^hb_", self: raw } + - { target: "^AST_", self: raw } + - { target: "^parser_", self: raw } + - { target: "char$", self: raw } + + PtrTips: + function: + - { target: "^herb_lex$", tips: [0, ref] } + - { target: "^herb_lex_file$", tips: [0, ref] } + - { target: "^herb_lex_to_buffer$", tips: [0, ref] } + - { target: "^herb_parse$", tips: [0, ref, ref] } + - { target: "^herb_version$", tips: [ref] } + - { target: "^herb_prism_version$", tips: [ref] } + - { target: "^herb_free_tokens$", tips: [ref] } + - { target: "^herb_extract_ruby_to_buffer$", tips: [0, ref] } + - { target: "^herb_extract_html_to_buffer$", tips: [0, ref] } + - { target: "^herb_extract_ruby_with_semicolons$", tips: [0, ref, ref] } + - { target: "^herb_extract_ruby_to_buffer_with_semicolons$", tips: [0, ref] } + - { target: "^herb_extract$", tips: [0, ref, ref] } + - { target: "^herb_extract_from_file$", tips: [0, ref, ref] } diff --git a/go/herb/cgo_helpers.go b/go/herb/cgo_helpers.go new file mode 100644 index 000000000..8ddbfbd60 --- /dev/null +++ b/go/herb/cgo_helpers.go @@ -0,0 +1,660 @@ +// MIT + +// WARNING: This file has automatically been generated on Mon, 10 Nov 2025 02:26:35 CST. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package herb + +/* +#cgo CFLAGS: -I.. -I../src/include -I../src +#cgo LDFLAGS: ${SRCDIR}/../../build/libherb.a ${SRCDIR}/../../vendor/prism/build/libprism.a +#include "herb_go.h" +#include +#include "cgo_helpers.h" +*/ +import "C" +import ( + "fmt" + "runtime" + "sync" + "unsafe" +) + +// Ref returns a reference to C object as it is. +func (x *hbarray) Ref() *C.hb_array_T { + if x == nil { + return nil + } + return (*C.hb_array_T)(unsafe.Pointer(x)) +} + +// Free cleanups the referenced memory using C free. +func (x *hbarray) Free() { + if x != nil { + C.free(unsafe.Pointer(x)) + } +} + +// NewhbarrayRef converts the C object reference into a raw struct reference without wrapping. +func NewhbarrayRef(ref unsafe.Pointer) *hbarray { + return (*hbarray)(ref) +} + +// Newhbarray allocates a new C object of this type and converts the reference into +// a raw struct reference without wrapping. +func Newhbarray() *hbarray { + return (*hbarray)(allocHbarrayMemory(1)) +} + +// allocHbarrayMemory allocates memory for type C.hb_array_T in C. +// The caller is responsible for freeing the this memory via C.free. +func allocHbarrayMemory(n int) unsafe.Pointer { + mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOfHbarrayValue)) + if mem == nil { + panic(fmt.Sprintln("memory alloc error: ", err)) + } + return mem +} + +const sizeOfHbarrayValue = unsafe.Sizeof([1]C.hb_array_T{}) + +// cgoAllocMap stores pointers to C allocated memory for future reference. +type cgoAllocMap struct { + mux sync.RWMutex + m map[unsafe.Pointer]struct{} +} + +var cgoAllocsUnknown = new(cgoAllocMap) + +func (a *cgoAllocMap) Add(ptr unsafe.Pointer) { + a.mux.Lock() + if a.m == nil { + a.m = make(map[unsafe.Pointer]struct{}) + } + a.m[ptr] = struct{}{} + a.mux.Unlock() +} + +func (a *cgoAllocMap) IsEmpty() bool { + a.mux.RLock() + isEmpty := len(a.m) == 0 + a.mux.RUnlock() + return isEmpty +} + +func (a *cgoAllocMap) Borrow(b *cgoAllocMap) { + if b == nil || b.IsEmpty() { + return + } + b.mux.Lock() + a.mux.Lock() + for ptr := range b.m { + if a.m == nil { + a.m = make(map[unsafe.Pointer]struct{}) + } + a.m[ptr] = struct{}{} + delete(b.m, ptr) + } + a.mux.Unlock() + b.mux.Unlock() +} + +func (a *cgoAllocMap) Free() { + a.mux.Lock() + for ptr := range a.m { + C.free(ptr) + delete(a.m, ptr) + } + a.mux.Unlock() +} + +// PassRef returns a reference to C object as it is or allocates a new C object of this type. +func (x *hbarray) PassRef() *C.hb_array_T { + if x == nil { + x = (*hbarray)(allocHbarrayMemory(1)) + } + return (*C.hb_array_T)(unsafe.Pointer(x)) +} + +// Ref returns a reference to C object as it is. +func (x *hbbuffer) Ref() *C.hb_buffer_T { + if x == nil { + return nil + } + return (*C.hb_buffer_T)(unsafe.Pointer(x)) +} + +// Free cleanups the referenced memory using C free. +func (x *hbbuffer) Free() { + if x != nil { + C.free(unsafe.Pointer(x)) + } +} + +// NewhbbufferRef converts the C object reference into a raw struct reference without wrapping. +func NewhbbufferRef(ref unsafe.Pointer) *hbbuffer { + return (*hbbuffer)(ref) +} + +// Newhbbuffer allocates a new C object of this type and converts the reference into +// a raw struct reference without wrapping. +func Newhbbuffer() *hbbuffer { + return (*hbbuffer)(allocHbbufferMemory(1)) +} + +// allocHbbufferMemory allocates memory for type C.hb_buffer_T in C. +// The caller is responsible for freeing the this memory via C.free. +func allocHbbufferMemory(n int) unsafe.Pointer { + mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOfHbbufferValue)) + if mem == nil { + panic(fmt.Sprintln("memory alloc error: ", err)) + } + return mem +} + +const sizeOfHbbufferValue = unsafe.Sizeof([1]C.hb_buffer_T{}) + +// PassRef returns a reference to C object as it is or allocates a new C object of this type. +func (x *hbbuffer) PassRef() *C.hb_buffer_T { + if x == nil { + x = (*hbbuffer)(allocHbbufferMemory(1)) + } + return (*C.hb_buffer_T)(unsafe.Pointer(x)) +} + +// Ref returns a reference to C object as it is. +func (x *ASTDOCUMENTNODE) Ref() *C.AST_DOCUMENT_NODE_T { + if x == nil { + return nil + } + return (*C.AST_DOCUMENT_NODE_T)(unsafe.Pointer(x)) +} + +// Free cleanups the referenced memory using C free. +func (x *ASTDOCUMENTNODE) Free() { + if x != nil { + C.free(unsafe.Pointer(x)) + } +} + +// NewASTDOCUMENTNODERef converts the C object reference into a raw struct reference without wrapping. +func NewASTDOCUMENTNODERef(ref unsafe.Pointer) *ASTDOCUMENTNODE { + return (*ASTDOCUMENTNODE)(ref) +} + +// NewASTDOCUMENTNODE allocates a new C object of this type and converts the reference into +// a raw struct reference without wrapping. +func NewASTDOCUMENTNODE() *ASTDOCUMENTNODE { + return (*ASTDOCUMENTNODE)(allocASTDOCUMENTNODEMemory(1)) +} + +// allocASTDOCUMENTNODEMemory allocates memory for type C.AST_DOCUMENT_NODE_T in C. +// The caller is responsible for freeing the this memory via C.free. +func allocASTDOCUMENTNODEMemory(n int) unsafe.Pointer { + mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOfASTDOCUMENTNODEValue)) + if mem == nil { + panic(fmt.Sprintln("memory alloc error: ", err)) + } + return mem +} + +const sizeOfASTDOCUMENTNODEValue = unsafe.Sizeof([1]C.AST_DOCUMENT_NODE_T{}) + +// PassRef returns a reference to C object as it is or allocates a new C object of this type. +func (x *ASTDOCUMENTNODE) PassRef() *C.AST_DOCUMENT_NODE_T { + if x == nil { + x = (*ASTDOCUMENTNODE)(allocASTDOCUMENTNODEMemory(1)) + } + return (*C.AST_DOCUMENT_NODE_T)(unsafe.Pointer(x)) +} + +// Ref returns a reference to C object as it is. +func (x *parseroptions) Ref() *C.parser_options_T { + if x == nil { + return nil + } + return (*C.parser_options_T)(unsafe.Pointer(x)) +} + +// Free cleanups the referenced memory using C free. +func (x *parseroptions) Free() { + if x != nil { + C.free(unsafe.Pointer(x)) + } +} + +// NewparseroptionsRef converts the C object reference into a raw struct reference without wrapping. +func NewparseroptionsRef(ref unsafe.Pointer) *parseroptions { + return (*parseroptions)(ref) +} + +// Newparseroptions allocates a new C object of this type and converts the reference into +// a raw struct reference without wrapping. +func Newparseroptions() *parseroptions { + return (*parseroptions)(allocParseroptionsMemory(1)) +} + +// allocParseroptionsMemory allocates memory for type C.parser_options_T in C. +// The caller is responsible for freeing the this memory via C.free. +func allocParseroptionsMemory(n int) unsafe.Pointer { + mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOfParseroptionsValue)) + if mem == nil { + panic(fmt.Sprintln("memory alloc error: ", err)) + } + return mem +} + +const sizeOfParseroptionsValue = unsafe.Sizeof([1]C.parser_options_T{}) + +// PassRef returns a reference to C object as it is or allocates a new C object of this type. +func (x *parseroptions) PassRef() *C.parser_options_T { + if x == nil { + x = (*parseroptions)(allocParseroptionsMemory(1)) + } + return (*C.parser_options_T)(unsafe.Pointer(x)) +} + +// allocPositionMemory allocates memory for type C.position_T in C. +// The caller is responsible for freeing the this memory via C.free. +func allocPositionMemory(n int) unsafe.Pointer { + mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOfPositionValue)) + if mem == nil { + panic(fmt.Sprintln("memory alloc error: ", err)) + } + return mem +} + +const sizeOfPositionValue = unsafe.Sizeof([1]C.position_T{}) + +// Ref returns the underlying reference to C object or nil if struct is nil. +func (x *position) Ref() *C.position_T { + if x == nil { + return nil + } + return x.refde50b33c +} + +// Free invokes alloc map's free mechanism that cleanups any allocated memory using C free. +// Does nothing if struct is nil or has no allocation map. +func (x *position) Free() { + if x != nil && x.allocsde50b33c != nil { + x.allocsde50b33c.(*cgoAllocMap).Free() + x.refde50b33c = nil + } +} + +// NewpositionRef creates a new wrapper struct with underlying reference set to the original C object. +// Returns nil if the provided pointer to C object is nil too. +func NewpositionRef(ref unsafe.Pointer) *position { + if ref == nil { + return nil + } + obj := new(position) + obj.refde50b33c = (*C.position_T)(unsafe.Pointer(ref)) + return obj +} + +// PassRef returns the underlying C object, otherwise it will allocate one and set its values +// from this wrapping struct, counting allocations into an allocation map. +func (x *position) PassRef() (*C.position_T, *cgoAllocMap) { + if x == nil { + return nil, nil + } else if x.refde50b33c != nil { + return x.refde50b33c, nil + } + memde50b33c := allocPositionMemory(1) + refde50b33c := (*C.position_T)(memde50b33c) + allocsde50b33c := new(cgoAllocMap) + allocsde50b33c.Add(memde50b33c) + + var cline_allocs *cgoAllocMap + refde50b33c.line, cline_allocs = (C.uint)(x.line), cgoAllocsUnknown + allocsde50b33c.Borrow(cline_allocs) + + var ccolumn_allocs *cgoAllocMap + refde50b33c.column, ccolumn_allocs = (C.uint)(x.column), cgoAllocsUnknown + allocsde50b33c.Borrow(ccolumn_allocs) + + x.refde50b33c = refde50b33c + x.allocsde50b33c = allocsde50b33c + return refde50b33c, allocsde50b33c + +} + +// PassValue does the same as PassRef except that it will try to dereference the returned pointer. +func (x position) PassValue() (C.position_T, *cgoAllocMap) { + if x.refde50b33c != nil { + return *x.refde50b33c, nil + } + ref, allocs := x.PassRef() + return *ref, allocs +} + +// Deref uses the underlying reference to C object and fills the wrapping struct with values. +// Do not forget to call this method whether you get a struct for C object and want to read its values. +func (x *position) Deref() { + if x.refde50b33c == nil { + return + } + x.line = (uint32)(x.refde50b33c.line) + x.column = (uint32)(x.refde50b33c.column) +} + +// alloc_rangeMemory allocates memory for type C.range_T in C. +// The caller is responsible for freeing the this memory via C.free. +func alloc_rangeMemory(n int) unsafe.Pointer { + mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOf_rangeValue)) + if mem == nil { + panic(fmt.Sprintln("memory alloc error: ", err)) + } + return mem +} + +const sizeOf_rangeValue = unsafe.Sizeof([1]C.range_T{}) + +// Ref returns the underlying reference to C object or nil if struct is nil. +func (x *_range) Ref() *C.range_T { + if x == nil { + return nil + } + return x.refbaf919a8 +} + +// Free invokes alloc map's free mechanism that cleanups any allocated memory using C free. +// Does nothing if struct is nil or has no allocation map. +func (x *_range) Free() { + if x != nil && x.allocsbaf919a8 != nil { + x.allocsbaf919a8.(*cgoAllocMap).Free() + x.refbaf919a8 = nil + } +} + +// New_rangeRef creates a new wrapper struct with underlying reference set to the original C object. +// Returns nil if the provided pointer to C object is nil too. +func New_rangeRef(ref unsafe.Pointer) *_range { + if ref == nil { + return nil + } + obj := new(_range) + obj.refbaf919a8 = (*C.range_T)(unsafe.Pointer(ref)) + return obj +} + +// PassRef returns the underlying C object, otherwise it will allocate one and set its values +// from this wrapping struct, counting allocations into an allocation map. +func (x *_range) PassRef() (*C.range_T, *cgoAllocMap) { + if x == nil { + return nil, nil + } else if x.refbaf919a8 != nil { + return x.refbaf919a8, nil + } + membaf919a8 := alloc_rangeMemory(1) + refbaf919a8 := (*C.range_T)(membaf919a8) + allocsbaf919a8 := new(cgoAllocMap) + allocsbaf919a8.Add(membaf919a8) + + var cfrom_allocs *cgoAllocMap + refbaf919a8.from, cfrom_allocs = (C.uint)(x.from), cgoAllocsUnknown + allocsbaf919a8.Borrow(cfrom_allocs) + + var cto_allocs *cgoAllocMap + refbaf919a8.to, cto_allocs = (C.uint)(x.to), cgoAllocsUnknown + allocsbaf919a8.Borrow(cto_allocs) + + x.refbaf919a8 = refbaf919a8 + x.allocsbaf919a8 = allocsbaf919a8 + return refbaf919a8, allocsbaf919a8 + +} + +// PassValue does the same as PassRef except that it will try to dereference the returned pointer. +func (x _range) PassValue() (C.range_T, *cgoAllocMap) { + if x.refbaf919a8 != nil { + return *x.refbaf919a8, nil + } + ref, allocs := x.PassRef() + return *ref, allocs +} + +// Deref uses the underlying reference to C object and fills the wrapping struct with values. +// Do not forget to call this method whether you get a struct for C object and want to read its values. +func (x *_range) Deref() { + if x.refbaf919a8 == nil { + return + } + x.from = (uint32)(x.refbaf919a8.from) + x.to = (uint32)(x.refbaf919a8.to) +} + +// allocLocationMemory allocates memory for type C.location_T in C. +// The caller is responsible for freeing the this memory via C.free. +func allocLocationMemory(n int) unsafe.Pointer { + mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOfLocationValue)) + if mem == nil { + panic(fmt.Sprintln("memory alloc error: ", err)) + } + return mem +} + +const sizeOfLocationValue = unsafe.Sizeof([1]C.location_T{}) + +// Ref returns the underlying reference to C object or nil if struct is nil. +func (x *location) Ref() *C.location_T { + if x == nil { + return nil + } + return x.refac96ad16 +} + +// Free invokes alloc map's free mechanism that cleanups any allocated memory using C free. +// Does nothing if struct is nil or has no allocation map. +func (x *location) Free() { + if x != nil && x.allocsac96ad16 != nil { + x.allocsac96ad16.(*cgoAllocMap).Free() + x.refac96ad16 = nil + } +} + +// NewlocationRef creates a new wrapper struct with underlying reference set to the original C object. +// Returns nil if the provided pointer to C object is nil too. +func NewlocationRef(ref unsafe.Pointer) *location { + if ref == nil { + return nil + } + obj := new(location) + obj.refac96ad16 = (*C.location_T)(unsafe.Pointer(ref)) + return obj +} + +// PassRef returns the underlying C object, otherwise it will allocate one and set its values +// from this wrapping struct, counting allocations into an allocation map. +func (x *location) PassRef() (*C.location_T, *cgoAllocMap) { + if x == nil { + return nil, nil + } else if x.refac96ad16 != nil { + return x.refac96ad16, nil + } + memac96ad16 := allocLocationMemory(1) + refac96ad16 := (*C.location_T)(memac96ad16) + allocsac96ad16 := new(cgoAllocMap) + allocsac96ad16.Add(memac96ad16) + + var cstart_allocs *cgoAllocMap + refac96ad16.start, cstart_allocs = x.start.PassValue() + allocsac96ad16.Borrow(cstart_allocs) + + var cend_allocs *cgoAllocMap + refac96ad16.end, cend_allocs = x.end.PassValue() + allocsac96ad16.Borrow(cend_allocs) + + x.refac96ad16 = refac96ad16 + x.allocsac96ad16 = allocsac96ad16 + return refac96ad16, allocsac96ad16 + +} + +// PassValue does the same as PassRef except that it will try to dereference the returned pointer. +func (x location) PassValue() (C.location_T, *cgoAllocMap) { + if x.refac96ad16 != nil { + return *x.refac96ad16, nil + } + ref, allocs := x.PassRef() + return *ref, allocs +} + +// Deref uses the underlying reference to C object and fills the wrapping struct with values. +// Do not forget to call this method whether you get a struct for C object and want to read its values. +func (x *location) Deref() { + if x.refac96ad16 == nil { + return + } + x.start = *NewpositionRef(unsafe.Pointer(&x.refac96ad16.start)) + x.end = *NewpositionRef(unsafe.Pointer(&x.refac96ad16.end)) +} + +// allocTokenMemory allocates memory for type C.token_T in C. +// The caller is responsible for freeing the this memory via C.free. +func allocTokenMemory(n int) unsafe.Pointer { + mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOfTokenValue)) + if mem == nil { + panic(fmt.Sprintln("memory alloc error: ", err)) + } + return mem +} + +const sizeOfTokenValue = unsafe.Sizeof([1]C.token_T{}) + +// Ref returns the underlying reference to C object or nil if struct is nil. +func (x *token) Ref() *C.token_T { + if x == nil { + return nil + } + return x.ref8d9fe5f8 +} + +// Free invokes alloc map's free mechanism that cleanups any allocated memory using C free. +// Does nothing if struct is nil or has no allocation map. +func (x *token) Free() { + if x != nil && x.allocs8d9fe5f8 != nil { + x.allocs8d9fe5f8.(*cgoAllocMap).Free() + x.ref8d9fe5f8 = nil + } +} + +// NewtokenRef creates a new wrapper struct with underlying reference set to the original C object. +// Returns nil if the provided pointer to C object is nil too. +func NewtokenRef(ref unsafe.Pointer) *token { + if ref == nil { + return nil + } + obj := new(token) + obj.ref8d9fe5f8 = (*C.token_T)(unsafe.Pointer(ref)) + return obj +} + +// PassRef returns the underlying C object, otherwise it will allocate one and set its values +// from this wrapping struct, counting allocations into an allocation map. +func (x *token) PassRef() (*C.token_T, *cgoAllocMap) { + if x == nil { + return nil, nil + } else if x.ref8d9fe5f8 != nil { + return x.ref8d9fe5f8, nil + } + mem8d9fe5f8 := allocTokenMemory(1) + ref8d9fe5f8 := (*C.token_T)(mem8d9fe5f8) + allocs8d9fe5f8 := new(cgoAllocMap) + allocs8d9fe5f8.Add(mem8d9fe5f8) + + var cvalue_allocs *cgoAllocMap + ref8d9fe5f8.value, cvalue_allocs = *(**C.char)(unsafe.Pointer(&x.value)), cgoAllocsUnknown + allocs8d9fe5f8.Borrow(cvalue_allocs) + + var c_range_allocs *cgoAllocMap + ref8d9fe5f8._range, c_range_allocs = x._range.PassValue() + allocs8d9fe5f8.Borrow(c_range_allocs) + + var clocation_allocs *cgoAllocMap + ref8d9fe5f8.location, clocation_allocs = x.location.PassValue() + allocs8d9fe5f8.Borrow(clocation_allocs) + + var c_type_allocs *cgoAllocMap + ref8d9fe5f8._type, c_type_allocs = (C.token_type_T)(x.kind), cgoAllocsUnknown + allocs8d9fe5f8.Borrow(c_type_allocs) + + x.ref8d9fe5f8 = ref8d9fe5f8 + x.allocs8d9fe5f8 = allocs8d9fe5f8 + return ref8d9fe5f8, allocs8d9fe5f8 + +} + +// PassValue does the same as PassRef except that it will try to dereference the returned pointer. +func (x token) PassValue() (C.token_T, *cgoAllocMap) { + if x.ref8d9fe5f8 != nil { + return *x.ref8d9fe5f8, nil + } + ref, allocs := x.PassRef() + return *ref, allocs +} + +// Deref uses the underlying reference to C object and fills the wrapping struct with values. +// Do not forget to call this method whether you get a struct for C object and want to read its values. +func (x *token) Deref() { + if x.ref8d9fe5f8 == nil { + return + } + x.value = (*byte)(unsafe.Pointer(x.ref8d9fe5f8.value)) + x._range = *New_rangeRef(unsafe.Pointer(&x.ref8d9fe5f8._range)) + x.location = *NewlocationRef(unsafe.Pointer(&x.ref8d9fe5f8.location)) + x.kind = (tokentype)(x.ref8d9fe5f8._type) +} + +// safeString ensures that the string is NULL-terminated, a NULL-terminated copy is created otherwise. +func safeString(str string) string { + if len(str) > 0 && str[len(str)-1] != '\x00' { + str = str + "\x00" + } else if len(str) == 0 { + str = "\x00" + } + return str +} + +// unpackPCharString copies the data from Go string as *C.char. +func unpackPCharString(str string) (*C.char, *cgoAllocMap) { + allocs := new(cgoAllocMap) + defer runtime.SetFinalizer(allocs, func(a *cgoAllocMap) { + go a.Free() + }) + + str = safeString(str) + mem0 := unsafe.Pointer(C.CString(str)) + allocs.Add(mem0) + return (*C.char)(mem0), allocs +} + +type stringHeader struct { + Data unsafe.Pointer + Len int +} + +type sliceHeader struct { + Data unsafe.Pointer + Len int + Cap int +} + +// copyPHbarrayBytes copies the data from Go slice as *C.hb_array_T. +func copyPHbarrayBytes(slice *sliceHeader) (*C.hb_array_T, *cgoAllocMap) { + allocs := new(cgoAllocMap) + defer runtime.SetFinalizer(allocs, func(a *cgoAllocMap) { + go a.Free() + }) + + mem0 := unsafe.Pointer(C.CBytes(*(*[]byte)(unsafe.Pointer(&sliceHeader{ + Data: slice.Data, + Len: int(sizeOfHbarrayValue) * slice.Len, + Cap: int(sizeOfHbarrayValue) * slice.Len, + })))) + allocs.Add(mem0) + + return (*C.hb_array_T)(mem0), allocs +} diff --git a/go/herb/cgo_helpers.h b/go/herb/cgo_helpers.h new file mode 100644 index 000000000..ee3e883bc --- /dev/null +++ b/go/herb/cgo_helpers.h @@ -0,0 +1,11 @@ +// MIT + +// WARNING: This file has automatically been generated on Mon, 10 Nov 2025 02:26:35 CST. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +#include "herb_go.h" +#include +#pragma once + +#define __CGOGEN 1 + diff --git a/go/herb/const.go b/go/herb/const.go new file mode 100644 index 000000000..be40cbfa6 --- /dev/null +++ b/go/herb/const.go @@ -0,0 +1,27 @@ +// MIT + +// WARNING: This file has automatically been generated on Mon, 10 Nov 2025 02:26:35 CST. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package herb + +/* +#cgo CFLAGS: -I.. -I../src/include -I../src +#cgo LDFLAGS: ${SRCDIR}/../../build/libherb.a ${SRCDIR}/../../vendor/prism/build/libprism.a +#include "herb_go.h" +#include +#include "cgo_helpers.h" +*/ +import "C" + +// tokentype as declared in herb/herb_go.h:43 +type tokentype int32 + +// tokentype enumeration from herb/herb_go.h:43 +const () + +// herbextractlanguage as declared in herb/herb_go.h:56 +type herbextractlanguage int32 + +// herbextractlanguage enumeration from herb/herb_go.h:56 +const () diff --git a/go/herb/doc.go b/go/herb/doc.go new file mode 100644 index 000000000..1902fc727 --- /dev/null +++ b/go/herb/doc.go @@ -0,0 +1,9 @@ +// MIT + +// WARNING: This file has automatically been generated on Mon, 10 Nov 2025 02:26:35 CST. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +/* +Go bindings for Herb - HTML-aware ERB parser +*/ +package herb diff --git a/go/herb/go.mod b/go/herb/go.mod new file mode 100644 index 000000000..38d15839e --- /dev/null +++ b/go/herb/go.mod @@ -0,0 +1,3 @@ +module github.com/marcoroth/herb + +go 1.24.7 diff --git a/go/herb/helpers.go b/go/herb/helpers.go new file mode 100644 index 000000000..2c15672c8 --- /dev/null +++ b/go/herb/helpers.go @@ -0,0 +1,17 @@ +package herb + +/* +#include "herb_go.h" +#include +*/ +import "C" + +// HerbVersion returns the herb library version as a string +func HerbVersion() string { + return C.GoString(C.herb_version()) +} + +// PrismVersion returns the prism library version as a string +func PrismVersion() string { + return C.GoString(C.herb_prism_version()) +} diff --git a/go/herb/herb.go b/go/herb/herb.go new file mode 100644 index 000000000..df01883e8 --- /dev/null +++ b/go/herb/herb.go @@ -0,0 +1,176 @@ +// MIT + +// WARNING: This file has automatically been generated on Mon, 10 Nov 2025 02:26:35 CST. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package herb + +/* +#cgo CFLAGS: -I.. -I../src/include -I../src +#cgo LDFLAGS: ${SRCDIR}/../../build/libherb.a ${SRCDIR}/../../vendor/prism/build/libprism.a +#include "herb_go.h" +#include +#include "cgo_helpers.h" +*/ +import "C" +import ( + "runtime" + "unsafe" +) + +// Lextobuffer function as declared in herb/herb_go.h:59 +func Lextobuffer(source string, output *hbbuffer) { + source = safeString(source) + csource, csourceAllocMap := unpackPCharString(source) + coutput, coutputAllocMap := (*C.hb_buffer_T)(unsafe.Pointer(output)), cgoAllocsUnknown + C.herb_lex_to_buffer(csource, coutput) + runtime.KeepAlive(coutputAllocMap) + runtime.KeepAlive(source) + runtime.KeepAlive(csourceAllocMap) +} + +// Lex function as declared in herb/herb_go.h:60 +func Lex(source string) *hbarray { + source = safeString(source) + csource, csourceAllocMap := unpackPCharString(source) + __ret := C.herb_lex(csource) + runtime.KeepAlive(source) + runtime.KeepAlive(csourceAllocMap) + __v := *(**hbarray)(unsafe.Pointer(&__ret)) + return __v +} + +// Lexfile function as declared in herb/herb_go.h:61 +func Lexfile(path string) *hbarray { + path = safeString(path) + cpath, cpathAllocMap := unpackPCharString(path) + __ret := C.herb_lex_file(cpath) + runtime.KeepAlive(path) + runtime.KeepAlive(cpathAllocMap) + __v := *(**hbarray)(unsafe.Pointer(&__ret)) + return __v +} + +// Parse function as declared in herb/herb_go.h:62 +func Parse(source string, options *parseroptions) *ASTDOCUMENTNODE { + source = safeString(source) + csource, csourceAllocMap := unpackPCharString(source) + coptions, coptionsAllocMap := (*C.parser_options_T)(unsafe.Pointer(options)), cgoAllocsUnknown + __ret := C.herb_parse(csource, coptions) + runtime.KeepAlive(coptionsAllocMap) + runtime.KeepAlive(source) + runtime.KeepAlive(csourceAllocMap) + __v := *(**ASTDOCUMENTNODE)(unsafe.Pointer(&__ret)) + return __v +} + +// Version function as declared in herb/herb_go.h:63 +func Version() *byte { + __ret := C.herb_version() + __v := *(**byte)(unsafe.Pointer(&__ret)) + return __v +} + +// Prismversion function as declared in herb/herb_go.h:64 +func Prismversion() *byte { + __ret := C.herb_prism_version() + __v := *(**byte)(unsafe.Pointer(&__ret)) + return __v +} + +// Freetokens function as declared in herb/herb_go.h:65 +func Freetokens(tokens []*hbarray) { + ctokens, ctokensAllocMap := (**C.hb_array_T)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&tokens)).Data)), cgoAllocsUnknown + C.herb_free_tokens(ctokens) + runtime.KeepAlive(ctokensAllocMap) +} + +// Extractrubytobuffer function as declared in herb/herb_go.h:68 +func Extractrubytobuffer(source string, output *hbbuffer) { + source = safeString(source) + csource, csourceAllocMap := unpackPCharString(source) + coutput, coutputAllocMap := (*C.hb_buffer_T)(unsafe.Pointer(output)), cgoAllocsUnknown + C.herb_extract_ruby_to_buffer(csource, coutput) + runtime.KeepAlive(coutputAllocMap) + runtime.KeepAlive(source) + runtime.KeepAlive(csourceAllocMap) +} + +// Extracthtmltobuffer function as declared in herb/herb_go.h:69 +func Extracthtmltobuffer(source string, output *hbbuffer) { + source = safeString(source) + csource, csourceAllocMap := unpackPCharString(source) + coutput, coutputAllocMap := (*C.hb_buffer_T)(unsafe.Pointer(output)), cgoAllocsUnknown + C.herb_extract_html_to_buffer(csource, coutput) + runtime.KeepAlive(coutputAllocMap) + runtime.KeepAlive(source) + runtime.KeepAlive(csourceAllocMap) +} + +// Extractrubywithsemicolons function as declared in herb/herb_go.h:70 +func Extractrubywithsemicolons(source string) *byte { + source = safeString(source) + csource, csourceAllocMap := unpackPCharString(source) + __ret := C.herb_extract_ruby_with_semicolons(csource) + runtime.KeepAlive(source) + runtime.KeepAlive(csourceAllocMap) + __v := *(**byte)(unsafe.Pointer(&__ret)) + return __v +} + +// Extractrubytobufferwithsemicolons function as declared in herb/herb_go.h:71 +func Extractrubytobufferwithsemicolons(source string, output *hbbuffer) { + source = safeString(source) + csource, csourceAllocMap := unpackPCharString(source) + coutput, coutputAllocMap := (*C.hb_buffer_T)(unsafe.Pointer(output)), cgoAllocsUnknown + C.herb_extract_ruby_to_buffer_with_semicolons(csource, coutput) + runtime.KeepAlive(coutputAllocMap) + runtime.KeepAlive(source) + runtime.KeepAlive(csourceAllocMap) +} + +// Extract function as declared in herb/herb_go.h:72 +func Extract(source string, language herbextractlanguage) *byte { + source = safeString(source) + csource, csourceAllocMap := unpackPCharString(source) + clanguage, clanguageAllocMap := (C.herb_extract_language_T)(language), cgoAllocsUnknown + __ret := C.herb_extract(csource, clanguage) + runtime.KeepAlive(clanguageAllocMap) + runtime.KeepAlive(source) + runtime.KeepAlive(csourceAllocMap) + __v := *(**byte)(unsafe.Pointer(&__ret)) + return __v +} + +// Extractfromfile function as declared in herb/herb_go.h:73 +func Extractfromfile(path string, language herbextractlanguage) *byte { + path = safeString(path) + cpath, cpathAllocMap := unpackPCharString(path) + clanguage, clanguageAllocMap := (C.herb_extract_language_T)(language), cgoAllocsUnknown + __ret := C.herb_extract_from_file(cpath, clanguage) + runtime.KeepAlive(clanguageAllocMap) + runtime.KeepAlive(path) + runtime.KeepAlive(cpathAllocMap) + __v := *(**byte)(unsafe.Pointer(&__ret)) + return __v +} + +// Hbarraysize function as declared in herb/herb_go.h:76 +func Hbarraysize(array []hbarray) uint64 { + carray, carrayAllocMap := (*C.hb_array_T)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&array)).Data)), cgoAllocsUnknown + __ret := C.hb_array_size(carray) + runtime.KeepAlive(carrayAllocMap) + __v := (uint64)(__ret) + return __v +} + +// Hbarrayget function as declared in herb/herb_go.h:77 +func Hbarrayget(array []hbarray, index uint64) unsafe.Pointer { + carray, carrayAllocMap := (*C.hb_array_T)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&array)).Data)), cgoAllocsUnknown + cindex, cindexAllocMap := (C.ulong)(index), cgoAllocsUnknown + __ret := C.hb_array_get(carray, cindex) + runtime.KeepAlive(cindexAllocMap) + runtime.KeepAlive(carrayAllocMap) + __v := *(*unsafe.Pointer)(unsafe.Pointer(&__ret)) + return __v +} diff --git a/go/herb/herb_go.h b/go/herb/herb_go.h new file mode 100644 index 000000000..c33120eb3 --- /dev/null +++ b/go/herb/herb_go.h @@ -0,0 +1,86 @@ +// Go bindings wrapper - mirrors Java approach (herb_jni.h pattern) +// This header exposes herb's public API without including prism.h +#ifndef HERB_GO_H +#define HERB_GO_H + +#ifdef __cplusplus +extern "C" { +#endif + +// Forward declarations - opaque types for Go (no struct definitions) +typedef struct hb_array_T hb_array_T; +typedef struct hb_buffer_T hb_buffer_T; +typedef struct AST_DOCUMENT_NODE_STRUCT AST_DOCUMENT_NODE_T; +typedef struct parser_options_T parser_options_T; + +// Token structures (needed for inspection) +typedef struct POSITION_STRUCT { + unsigned int line; + unsigned int column; +} position_T; + +typedef struct RANGE_STRUCT { + unsigned int from; + unsigned int to; +} range_T; + +typedef struct LOCATION_STRUCT { + position_T start; + position_T end; +} location_T; + +typedef enum { + TOKEN_WHITESPACE, TOKEN_NBSP, TOKEN_NEWLINE, TOKEN_IDENTIFIER, + TOKEN_HTML_DOCTYPE, TOKEN_XML_DECLARATION, TOKEN_XML_DECLARATION_END, + TOKEN_CDATA_START, TOKEN_CDATA_END, + TOKEN_HTML_TAG_START, TOKEN_HTML_TAG_START_CLOSE, TOKEN_HTML_TAG_END, TOKEN_HTML_TAG_SELF_CLOSE, + TOKEN_HTML_COMMENT_START, TOKEN_HTML_COMMENT_END, + TOKEN_ERB_START, TOKEN_ERB_CONTENT, TOKEN_ERB_END, + TOKEN_LT, TOKEN_SLASH, TOKEN_EQUALS, TOKEN_QUOTE, TOKEN_BACKTICK, TOKEN_BACKSLASH, + TOKEN_DASH, TOKEN_UNDERSCORE, TOKEN_EXCLAMATION, TOKEN_SEMICOLON, TOKEN_COLON, + TOKEN_AT, TOKEN_PERCENT, TOKEN_AMPERSAND, + TOKEN_CHARACTER, TOKEN_ERROR, TOKEN_EOF, +} token_type_T; + +typedef struct TOKEN_STRUCT { + char* value; + range_T range; + location_T location; + token_type_T type; +} token_T; + +// Extract language enum (from extract.h) +typedef enum { + HERB_EXTRACT_LANGUAGE_RUBY, + HERB_EXTRACT_LANGUAGE_HTML, +} herb_extract_language_T; + +// Public herb API (from herb.h) +void herb_lex_to_buffer(const char* source, hb_buffer_T* output); +hb_array_T* herb_lex(const char* source); +hb_array_T* herb_lex_file(const char* path); +AST_DOCUMENT_NODE_T* herb_parse(const char* source, parser_options_T* options); +const char* herb_version(void); +const char* herb_prism_version(void); +void herb_free_tokens(hb_array_T** tokens); + +// Extract API (from extract.h) +void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output); +void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output); +char* herb_extract_ruby_with_semicolons(const char* source); +void herb_extract_ruby_to_buffer_with_semicolons(const char* source, hb_buffer_T* output); +char* herb_extract(const char* source, herb_extract_language_T language); +char* herb_extract_from_file(const char* path, herb_extract_language_T language); + +// Array helper functions +unsigned long hb_array_size(const hb_array_T* array); +void* hb_array_get(const hb_array_T* array, unsigned long index); + +// Token helper functions +const char* token_type_to_string(token_type_T type); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/go/herb/herb_test.go b/go/herb/herb_test.go new file mode 100644 index 000000000..4904e1109 --- /dev/null +++ b/go/herb/herb_test.go @@ -0,0 +1,65 @@ +package herb + +import ( + "testing" +) + +func TestVersion(t *testing.T) { + version := HerbVersion() + if version == "" { + t.Fatal("HerbVersion() returned empty string") + } + t.Logf("Herb version: %s", version) +} + +func TestPrismVersion(t *testing.T) { + version := PrismVersion() + if version == "" { + t.Fatal("PrismVersion() returned empty string") + } + t.Logf("Prism version: %s", version) +} + +func TestLex(t *testing.T) { + tests := []struct { + name string + source string + }{ + {"simple HTML", "

Hello

"}, + {"simple ERB", "<%= 'Hello' %>"}, + {"mixed content", "

<%= 'Hello World' %>

"}, + {"multiple tags", "

<%= name %>

"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tokens := Lex(tt.source) + if tokens == nil { + t.Errorf("Lex(%q) returned nil", tt.source) + return + } + }) + } +} + +func TestParse(t *testing.T) { + tests := []struct { + name string + source string + }{ + {"simple HTML", "

Hello

"}, + {"simple ERB", "<%= 'Hello' %>"}, + {"mixed content", "

<%= 'Hello World' %>

"}, + {"multiple tags", "

<%= name %>

"}, + {"if statement", "<% if true %>yes<% end %>"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ast := Parse(tt.source, nil) + if ast == nil { + t.Errorf("Parse(%q, nil) returned nil", tt.source) + } + }) + } +} diff --git a/go/herb/token_helpers.go b/go/herb/token_helpers.go new file mode 100644 index 000000000..86417bca1 --- /dev/null +++ b/go/herb/token_helpers.go @@ -0,0 +1,101 @@ +package herb + +/* +#include "herb_go.h" +#include +*/ +import "C" + +import ( + "fmt" + "strings" +) + +// TokenInfo represents a single token with all its information +type TokenInfo struct { + Type string + Value string + Range [2]uint + StartPos [2]uint // [line, column] + EndPos [2]uint // [line, column] +} + +// InspectTokens returns a formatted string representation of all tokens +// similar to Rust's inspect() method +func InspectTokens(tokens *hbarray) string { + if tokens == nil { + return "" + } + + // Call C functions directly + cArray := (*C.hb_array_T)(tokens) + size := uint64(C.hb_array_size(cArray)) + + if size == 0 { + return "" + } + + var result strings.Builder + for i := range size { + tokenPtr := C.hb_array_get(cArray, C.ulong(i)) + if tokenPtr == nil { + continue + } + + token := (*C.token_T)(tokenPtr) + tokenInfo := extractTokenInfo(token) + result.WriteString(formatToken(tokenInfo)) + if i < size-1 { + result.WriteString("\n") + } + } + + return result.String() +} + +// extractTokenInfo extracts token information from C token_T pointer +func extractTokenInfo(token *C.token_T) TokenInfo { + typeStr := C.GoString(C.token_type_to_string(token._type)) + + var value string + if token.value == nil { + value = "" + } else { + value = C.GoString(token.value) + } + + // Handle EOF special case + if typeStr == "TOKEN_EOF" && value == "" { + value = "" + } + + return TokenInfo{ + Type: typeStr, + Value: escapeValue(value), + Range: [2]uint{uint(token._range.from), uint(token._range.to)}, + StartPos: [2]uint{uint(token.location.start.line), uint(token.location.start.column)}, + EndPos: [2]uint{uint(token.location.end.line), uint(token.location.end.column)}, + } +} + +// escapeValue escapes special characters in token value +func escapeValue(value string) string { + value = strings.ReplaceAll(value, "\\", "\\\\") + value = strings.ReplaceAll(value, "\n", "\\n") + value = strings.ReplaceAll(value, "\r", "\\r") + value = strings.ReplaceAll(value, "\t", "\\t") + value = strings.ReplaceAll(value, "\"", "\\\"") + return value +} + +// formatToken formats a single token similar to Rust's inspect() +func formatToken(t TokenInfo) string { + return fmt.Sprintf( + `#`, + t.Type, + t.Value, + t.Range[0], t.Range[1], + t.StartPos[0], t.StartPos[1], + t.EndPos[0], t.EndPos[1], + ) +} diff --git a/go/herb/token_helpers_test.go b/go/herb/token_helpers_test.go new file mode 100644 index 000000000..7b32f1d35 --- /dev/null +++ b/go/herb/token_helpers_test.go @@ -0,0 +1,269 @@ +package herb + +import ( + "strings" + "testing" +) + +func TestInspectTokens_EmptyInput(t *testing.T) { + result := InspectTokens(nil) + if result != "" { + t.Errorf("InspectTokens(nil) = %q, want empty string", result) + } +} + +func TestInspectTokens_SimpleHTML(t *testing.T) { + source := "

Hello

" + tokens := Lex(source) + if tokens == nil { + t.Fatal("Lex() returned nil") + } + + result := InspectTokens(tokens) + if result == "" { + t.Error("InspectTokens() returned empty string for valid tokens") + } + + // Verify expected tokens are present + expectedTokens := []string{ + "TOKEN_HTML_TAG_START", + "TOKEN_IDENTIFIER", + "TOKEN_HTML_TAG_END", + "TOKEN_HTML_TAG_START_CLOSE", + "TOKEN_EOF", + } + + for _, expected := range expectedTokens { + if !strings.Contains(result, expected) { + t.Errorf("InspectTokens() output missing expected token type: %s", expected) + } + } + + // Verify token format structure + if !strings.Contains(result, "# value + if !strings.Contains(result, "TOKEN_EOF") { + t.Error("InspectTokens() missing EOF token") + } + if !strings.Contains(result, "") { + t.Error("InspectTokens() EOF token missing value") + } +} + +func TestInspectTokens_ValueEscaping(t *testing.T) { + tests := []struct { + name string + source string + contains string // escaped sequence we expect in output + }{ + { + name: "newline in content", + source: "hello\nworld", + contains: "\\n", + }, + { + name: "tab in content", + source: "hello\tworld", + contains: "\\t", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tokens := Lex(tt.source) + if tokens == nil { + t.Fatal("Lex() returned nil") + } + + result := InspectTokens(tokens) + if !strings.Contains(result, tt.contains) { + t.Errorf("InspectTokens() output missing escaped sequence %q", tt.contains) + } + }) + } +} + +func TestInspectTokens_RangeAccuracy(t *testing.T) { + source := "

" + tokens := Lex(source) + if tokens == nil { + t.Fatal("Lex() returned nil") + } + + result := InspectTokens(tokens) + + // First token '<' should be at range [0, 1] + if !strings.Contains(result, "range=[0, 1]") { + t.Error("InspectTokens() first token range incorrect, expected [0, 1]") + } + + // Second token 'h1' should be at range [1, 3] + if !strings.Contains(result, "range=[1, 3]") { + t.Error("InspectTokens() second token range incorrect, expected [1, 3]") + } + + // Third token '>' should be at range [3, 4] + if !strings.Contains(result, "range=[3, 4]") { + t.Error("InspectTokens() third token range incorrect, expected [3, 4]") + } +} + +func TestInspectTokens_LocationAccuracy(t *testing.T) { + source := "

" + tokens := Lex(source) + if tokens == nil { + t.Fatal("Lex() returned nil") + } + + result := InspectTokens(tokens) + + // All tokens should start at line 1 (1-indexed) + lines := strings.Split(result, "\n") + for i, line := range lines { + if line == "" { + continue + } + if !strings.Contains(line, "start=(1:") { + t.Errorf("Token %d should start at line 1, got: %s", i, line) + } + if !strings.Contains(line, "end=(1:") { + t.Errorf("Token %d should end at line 1, got: %s", i, line) + } + } +} + +func TestInspectTokens_MultilineContent(t *testing.T) { + source := "
\n

Hello

\n
" + tokens := Lex(source) + if tokens == nil { + t.Fatal("Lex() returned nil") + } + + result := InspectTokens(tokens) + + // Should have tokens on multiple lines + hasLine1 := strings.Contains(result, "start=(1:") + hasLine2 := strings.Contains(result, "start=(2:") + hasLine3 := strings.Contains(result, "start=(3:") + + if !hasLine1 || !hasLine2 || !hasLine3 { + t.Error("InspectTokens() should show tokens across multiple lines") + } +} + +func TestInspectTokens_ComplexERB(t *testing.T) { + source := "<% if true %>yes<% end %>" + tokens := Lex(source) + if tokens == nil { + t.Fatal("Lex() returned nil") + } + + result := InspectTokens(tokens) + + // Verify ERB control flow tokens + if !strings.Contains(result, "TOKEN_ERB_START") { + t.Error("InspectTokens() missing ERB start tokens") + } + if !strings.Contains(result, "TOKEN_ERB_CONTENT") { + t.Error("InspectTokens() missing ERB content tokens") + } + if !strings.Contains(result, "TOKEN_ERB_END") { + t.Error("InspectTokens() missing ERB end tokens") + } + + // Should have multiple ERB blocks (if and end) + erbStartCount := strings.Count(result, "TOKEN_ERB_START") + if erbStartCount < 2 { + t.Errorf("InspectTokens() found %d ERB start tokens, expected at least 2", erbStartCount) + } +} + +func TestInspectTokens_TokenValueQuoting(t *testing.T) { + source := `
` + tokens := Lex(source) + if tokens == nil { + t.Fatal("Lex() returned nil") + } + + result := InspectTokens(tokens) + + // Check that quote characters are properly escaped in output + if !strings.Contains(result, `TOKEN_QUOTE`) { + t.Error("InspectTokens() should detect quote tokens") + } +} diff --git a/go/herb/types.go b/go/herb/types.go new file mode 100644 index 000000000..b680a57d6 --- /dev/null +++ b/go/herb/types.go @@ -0,0 +1,70 @@ +// MIT + +// WARNING: This file has automatically been generated on Mon, 10 Nov 2025 02:26:35 CST. +// Code generated by https://git.io/c-for-go. DO NOT EDIT. + +package herb + +/* +#cgo CFLAGS: -I.. -I../src/include -I../src +#cgo LDFLAGS: ${SRCDIR}/../../build/libherb.a ${SRCDIR}/../../vendor/prism/build/libprism.a +#include "herb_go.h" +#include +#include "cgo_helpers.h" +*/ +import "C" + +// hbarray as declared in herb/herb_go.h:11 +type hbarray C.hb_array_T + +// hbbuffer as declared in herb/herb_go.h:12 +type hbbuffer C.hb_buffer_T + +// ASTDOCUMENTNODE as declared in herb/herb_go.h:13 +type ASTDOCUMENTNODE C.AST_DOCUMENT_NODE_T + +// parseroptions as declared in herb/herb_go.h:14 +type parseroptions C.parser_options_T + +// position as declared in herb/herb_go.h:20 +type position struct { + line uint32 + column uint32 + refde50b33c *C.position_T + allocsde50b33c interface{} +} + +// _range as declared in herb/herb_go.h:25 +type _range struct { + from uint32 + to uint32 + refbaf919a8 *C.range_T + allocsbaf919a8 interface{} +} + +// location as declared in herb/herb_go.h:30 +type location struct { + start position + end position + refac96ad16 *C.location_T + allocsac96ad16 interface{} +} + +// token as declared in herb/herb_go.h:50 +type token struct { + value *byte + _range _range + location location + kind tokentype + ref8d9fe5f8 *C.token_T + allocs8d9fe5f8 interface{} +} + +// _predefinedsize type as declared in go/:23 +type _predefinedsize uint64 + +// _predefinedwchar type as declared in go/:27 +type _predefinedwchar int32 + +// _predefinedptrdiff type as declared in go/:31 +type _predefinedptrdiff int64