Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ tools/python_bind/
Cypher → ANTLR Parser → Binder → Logical Plan → gopt Converter → Physical Plan → Execution
```

## Known Limitations

- **CSV-based bulk load does not support List type**: The CSV-based `COPY FROM` path does not support `List` type properties because Arrow CSV reader cannot natively parse list values from CSV. The Arrow RecordBatch-based bulk loader (used by the programmatic API) does support List type properties.

## Code Style

- **C++**: C++20, clang-format (style=file)
Expand Down
3 changes: 3 additions & 0 deletions doc/source/cypher_manual/ddl_clause.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ The following table lists the recommended syntax for defining default values for
| `DATE` | `prop DATE DEFAULT DATE('1970-01-01')` | `DATE('1970-01-01')` |
| `TIMESTAMP` | `prop TIMESTAMP DEFAULT TIMESTAMP('1970-01-01')` | `TIMESTAMP('1970-01-01')` |
| `INTERVAL` | `prop INTERVAL DEFAULT INTERVAL('0 year 0 month 0 day')` | `INTERVAL('0 year 0 month 0 day')` |
| `<type>[]` | `prop INT64[] DEFAULT [1, 2, 3]` | `[]` (empty list) |

List types are declared by appending `[]` to any supported element type (e.g., `STRING[]`, `DOUBLE[]`). Nesting is supported — `<type>[][]` declares a list of lists, and deeper nesting follows the same pattern. Default values use list literal syntax: `DEFAULT ['a', 'b']` or `DEFAULT [[1, 2], [3]]` for nested lists.

Please refer to the following examples for more usages.

Expand Down
3 changes: 3 additions & 0 deletions include/neug/common/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ struct DataType {

std::string ToString() const;

static std::string ToYAMLString(const DataType& type);
static DataType FromYAMLString(const std::string& str);

private:
DataTypeId id_;
std::shared_ptr<ExtraTypeInfo> type_info_;
Expand Down
6 changes: 4 additions & 2 deletions include/neug/compiler/gopt/g_ddl_converter.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "neug/compiler/gopt/g_catalog.h"
#include "neug/compiler/gopt/g_expr_converter.h"
#include "neug/compiler/gopt/g_type_converter.h"
#include "neug/compiler/main/client_context.h"
#include "neug/compiler/planner/operator/ddl/logical_alter.h"
#include "neug/compiler/planner/operator/ddl/logical_create_table.h"
#include "neug/compiler/planner/operator/ddl/logical_drop.h"
Expand All @@ -45,8 +46,9 @@ struct EdgeLabel {
class GDDLConverter {
public:
explicit GDDLConverter(std::shared_ptr<GAliasManager> aliasManager,
neug::catalog::Catalog* catalog)
: catalog{catalog}, exprConverter(aliasManager) {}
neug::catalog::Catalog* catalog,
main::ClientContext* clientContext)
: catalog{catalog}, exprConverter(aliasManager, clientContext) {}

virtual ~GDDLConverter() = default;

Expand Down
15 changes: 8 additions & 7 deletions include/neug/compiler/gopt/g_expr_converter.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "neug/compiler/gopt/g_precedence.h"
#include "neug/compiler/gopt/g_scalar_type.h"
#include "neug/compiler/gopt/g_type_converter.h"
#include "neug/compiler/main/client_context.h"
#include "neug/config.h"
#include "neug/generated/proto/plan/algebra.pb.h"
#include "neug/generated/proto/plan/common.pb.h"
Expand All @@ -44,8 +45,9 @@ namespace gopt {

class GExprConverter {
public:
GExprConverter(const std::shared_ptr<gopt::GAliasManager> aliasManager)
: aliasManager{std::move(aliasManager)} {}
GExprConverter(const std::shared_ptr<gopt::GAliasManager> aliasManager,
main::ClientContext* clientContext)
: aliasManager{std::move(aliasManager)}, ctx{clientContext} {}

// Main conversion function
std::unique_ptr<::common::Expression> convert(
Expand All @@ -61,7 +63,7 @@ class GExprConverter {
const binder::AggregateFunctionExpression& expr,
const planner::LogicalOperator& child);
std::unique_ptr<::common::Variable> convertDefaultVar();
std::unique_ptr<::common::Value> convertDefaultValue(
std::unique_ptr<::common::Expression> convertDefaultValue(
const binder::PropertyDefinition& propertyDef);
std::unique_ptr<::common::Property> convertPropertyExpr(
const std::string& propName);
Expand Down Expand Up @@ -122,7 +124,7 @@ class GExprConverter {
const std::vector<std::string>& schemaAlias);

// helper functions
std::unique_ptr<::common::Value> convertValue(
std::unique_ptr<::common::Expression> convertValue(
const neug::common::Value& value);
std::unique_ptr<::common::Variable> convertVarProperty(
const std::string& aliasName, const std::string& propertyName,
Expand All @@ -145,8 +147,6 @@ class GExprConverter {
std::unique_ptr<::common::Expression> convertUDFFunc(
const std::string& funcName, const binder::Expression& expr,
size_t paramNum, const std::vector<std::string>& schemaAlias);
std::unique_ptr<::common::Value> convertToLiteralArray(
const common::Value& value, const common::LogicalType& childType);
std::unique_ptr<::common::Expression> convertRegexFunc(
const binder::Expression& expr, const GScalarType& scalarType,
const std::vector<std::string>& schemaAlias);
Expand All @@ -156,13 +156,14 @@ class GExprConverter {
const binder::Expression& expr, const GScalarType& scalarType,
const std::vector<std::string>& schemaAlias);

std::unique_ptr<::common::Value> castLiteral(
std::unique_ptr<::common::Expression> castLiteral(
const binder::Expression& castExpr);

private:
const std::shared_ptr<gopt::GAliasManager> aliasManager;
gopt::GPhysicalTypeConverter typeConverter;
gopt::GPrecedence preced;
main::ClientContext* ctx;
};

} // namespace gopt
Expand Down
12 changes: 9 additions & 3 deletions include/neug/compiler/gopt/g_physical_convertor.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "neug/compiler/gopt/g_ddl_converter.h"
#include "neug/compiler/gopt/g_physical_analyzer.h"
#include "neug/compiler/gopt/g_query_converter.h"
#include "neug/compiler/main/client_context.h"
#include "neug/compiler/planner/operator/logical_plan.h"
#include "neug/compiler/planner/operator/simple/logical_extension.h"
#include "neug/generated/proto/plan/physical.pb.h"
Expand All @@ -29,8 +30,11 @@ namespace gopt {
class GPhysicalConvertor {
public:
GPhysicalConvertor(std::shared_ptr<GAliasManager> aliasManager,
neug::catalog::Catalog* catalog)
: aliasManager{aliasManager}, catalog{catalog} {}
neug::catalog::Catalog* catalog,
main::ClientContext* clientContext)
: aliasManager{aliasManager},
catalog{catalog},
clientContext{clientContext} {}

std::unique_ptr<::physical::PhysicalPlan> createEmptyPlan() {
auto physicalPlan = std::make_unique<::physical::PhysicalPlan>();
Expand Down Expand Up @@ -83,13 +87,15 @@ class GPhysicalConvertor {
private:
std::unique_ptr<::physical::PhysicalPlan> convertQuery(
const planner::LogicalPlan& plan, bool skipSink) {
auto converter = std::make_unique<GQueryConvertor>(aliasManager, catalog);
auto converter =
std::make_unique<GQueryConvertor>(aliasManager, catalog, clientContext);
return converter->convert(plan, skipSink);
}

private:
std::shared_ptr<GAliasManager> aliasManager;
neug::catalog::Catalog* catalog;
main::ClientContext* clientContext;
};

} // namespace gopt
Expand Down
5 changes: 4 additions & 1 deletion include/neug/compiler/gopt/g_query_converter.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "neug/compiler/gopt/g_ddl_converter.h"
#include "neug/compiler/gopt/g_expr_converter.h"
#include "neug/compiler/gopt/g_type_converter.h"
#include "neug/compiler/main/client_context.h"
#include "neug/compiler/planner/operator/extend/logical_extend.h"
#include "neug/compiler/planner/operator/extend/logical_recursive_extend.h"
#include "neug/compiler/planner/operator/logical_aggregate.h"
Expand Down Expand Up @@ -79,7 +80,8 @@ struct EdgeLabelId {
class GQueryConvertor {
public:
GQueryConvertor(std::shared_ptr<GAliasManager> aliasManager,
neug::catalog::Catalog* catalog);
neug::catalog::Catalog* catalog,
main::ClientContext* clientContext);

std::unique_ptr<::physical::PhysicalPlan> convert(
const planner::LogicalPlan& plan, bool skipSink);
Expand Down Expand Up @@ -256,6 +258,7 @@ class GQueryConvertor {
std::unique_ptr<GExprConverter> exprConvertor;
std::unique_ptr<GPhysicalTypeConverter> typeConverter;
neug::catalog::Catalog* catalog;
main::ClientContext* clientContext;
neug::gopt::GDDLConverter ddlConverter;
};

Expand Down
2 changes: 0 additions & 2 deletions include/neug/compiler/gopt/g_scalar_type.h
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,8 @@ class GScalarType {
} else if (func.name == function::ListCreationFunction::name) {
const auto& type = expr.getDataType();
if (type.getLogicalTypeID() == common::LogicalTypeID::LIST) {
LOG(INFO) << "type is list";
return ScalarType::TO_LIST;
} else if (type.getLogicalTypeID() == common::LogicalTypeID::STRUCT) {
LOG(INFO) << "type is struct";
return ScalarType::TO_TUPLE;
}
THROW_EXCEPTION_WITH_FILE_LINE("Invalid data type: " + type.toString() +
Expand Down
13 changes: 13 additions & 0 deletions include/neug/compiler/gopt/g_type_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ class GTypeUtils {
}
}
auto arrayType = node["array"];
if (!arrayType) {
arrayType = node["list"];
}
if (arrayType && arrayType.IsMap()) {
auto componentType = arrayType["component_type"];
CHECK(componentType.IsDefined())
Expand Down Expand Up @@ -147,6 +150,16 @@ class GTypeUtils {
return YAML_NODE_TEMPORAL_DATETIME();
case neug::common::LogicalTypeID::INTERVAL:
return YAML_NODE_TEMPORAL_INTERVAL();
case neug::common::LogicalTypeID::LIST: {
auto extraInfo = type.getExtraTypeInfo();
if (!extraInfo) {
THROW_RUNTIME_ERROR("List type should have extra info");
}
auto listType = extraInfo->constPtrCast<neug::common::ListTypeInfo>();
YAML::Node n;
n["array"]["component_type"] = toYAML(listType->getChildType());
return n;
}
default:
LOG(WARNING) << "Unsupported type in YAML: "
<< static_cast<uint8_t>(type.getLogicalTypeID());
Expand Down
44 changes: 44 additions & 0 deletions include/neug/storages/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,50 @@ Unlike the property table of vertices, the property table of edges is not column
Fow now, only one property is supported for edges, but developers can define a struct with multiple fields to store multiple properties.


## 5. List Property Storage

List properties are stored using a binary encoding in [VarLenColumn](../utils/property/column.h),
which extends the dual-buffer storage pattern used for strings. The encoding differs based on
whether element types are POD (fixed-size) or non-POD (variable-size like strings or nested lists).

### 5.1 Binary Encoding

**POD Lists** (integers, floats, dates, booleans):
```
┌──────────────────────────────────────────┐
│ T[0] T[1] … T[count-1] │
└──────────────────────────────────────────┘
Total bytes: count * sizeof(T) (empty list = 0 bytes)
```
`count` is not stored; it is derived from the buffer size as
`data_.size() / pod_type_size(child_id)`.

**Non-POD Lists** (strings, nested lists):
```
┌──────────┬──────────────────────────────┬────────────────────┐
│ count │ off[0] off[1] … off[count-1] │ data[0] data[1] … │
│ uint32_t │ uint32_t × count │ │
└──────────┴──────────────────────────────┴────────────────────┘
Empty list: 4 bytes (count=0).
```
`off[i]` is relative to the start of the data region (first byte after the
last offset entry). The data region size is derived from the buffer size:
`data_.size() - sizeof(uint32_t) - count * sizeof(uint32_t)`. The length of
element `i` is `off[i+1] - off[i]` for `i < count-1`, and
`data_region_size - off[count-1]` for the last element.

### 5.2 Thread Safety

VarLenColumn uses `std::atomic<size_t> pos_` for concurrent offset allocation via `fetch_add`.
- Concurrent writes to different indices are thread-safe
- The `insert_safe` parameter controls resize behavior: when `false`, throws on insufficient space;
when `true`, caller must provide external synchronization during resize

### 5.3 Nested Lists

Nested lists use non-POD encoding: each inner list is serialized as a blob, then stored
as a variable-length element in the outer list's data region.


## 6. Durability

Expand Down
7 changes: 7 additions & 0 deletions include/neug/storages/loader/loader_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,14 @@

#include <fstream>
#include <memory>
#include <shared_mutex>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>

#include "neug/common/types.h"
#include "neug/execution/common/types/value.h"
#include "neug/storages/loader/loading_config.h"
#include "neug/utils/exception/exception.h"
#include "neug/utils/string_utils.h"
Expand Down Expand Up @@ -244,4 +247,8 @@ void set_properties_column(std::shared_ptr<neug::ColumnBase> col,
const std::vector<vid_t>& vids,
std::shared_mutex& mutex);

execution::Value arrow_element_to_value(
const std::shared_ptr<arrow::Array>& arr, int64_t idx,
const DataType& neug_type);

} // namespace neug
1 change: 0 additions & 1 deletion include/neug/utils/id_indexer.h
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,6 @@ class LFIndexer {
while (true) {
INDEX_T ind = indices_ptr[index];
if (ind == LFIndexer<INDEX_T>::sentinel) {
VLOG(10) << "cannot find " << oid.to_string() << " in lf_indexer";
return ind;
} else if (keys_->get_any(ind) == oid) {
return ind;
Expand Down
7 changes: 7 additions & 0 deletions include/neug/utils/property/column.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@

namespace neug {
class Table;
class ModuleBroker;
class CheckpointManifest;

std::string_view truncate_utf8(std::string_view str, size_t length);

Expand All @@ -73,6 +75,11 @@ class ColumnBase : public Module {
virtual execution::Value get_any(size_t index) const = 0;

virtual void ingest(uint32_t index, OutArchive& arc) = 0;

virtual void DumpTo(Checkpoint& ckp, CheckpointManifest& meta,
const std::string& key);

virtual void RestoreChildren(ModuleBroker& store, const std::string& key) {}
};

template <typename T>
Expand Down
Loading
Loading