From 74b0911bf0b6dc2da7fee10b53929415a2b4fc20 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sat, 30 Aug 2025 15:40:36 +1000 Subject: [PATCH 01/22] Implement data storage API for ADAM --- src/ad4m/useADAM.tsx | 156 ++++++++++++++++++++++++++++++++++++++++++- src/api/CRUD.ts | 21 ++++++ 2 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 src/api/CRUD.ts diff --git a/src/ad4m/useADAM.tsx b/src/ad4m/useADAM.tsx index c123ac7..1e1aa62 100644 --- a/src/ad4m/useADAM.tsx +++ b/src/ad4m/useADAM.tsx @@ -1,7 +1,40 @@ -import { Ad4mClient, Agent } from '@coasys/ad4m' +import { Ad4mClient, Agent, Link, LinkMutations } from '@coasys/ad4m' import Ad4mConnectUI, { Ad4mConnectElement, getAd4mClient } from '@coasys/ad4m-connect' -import { defineState, getMutableState, useHookstate, useMutableState } from '@ir-engine/hyperflux' +import { defineState, getMutableState, getState, useHookstate, useMutableState } from '@ir-engine/hyperflux' import { useEffect } from 'react' +import { CRUD_API, P2P_API } from '../api/CRUD' + +// import { languages } from '@coasys/flux-constants' +const languages = { FILE_STORAGE_LANGUAGE: 'QmzSYwdjqeP9D13Sfmyc5HcabM9jL3DtPyhadnF6dQXu4FjVSbQ' } +const { FILE_STORAGE_LANGUAGE } = languages + +export const blobToDataURL = (blob: Blob): Promise => { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => { + const result = reader.result as string + const base64 = result.split(',')[1] + resolve(base64 as string) + } + reader.onerror = () => reject(reader.error) + reader.onabort = () => reject(new Error('Read aborted')) + reader.readAsDataURL(blob) + }) +} + +export const dataURLToBlob = (dataURL: string): Promise => { + return new Promise((resolve, reject) => { + const arr = dataURL.split(',') + const mime = arr[0].match(/:(.*?);/)?.[1] + const bstr = atob(arr[1]) + let n = bstr.length + const u8arr = new Uint8Array(n) + while (n--) { + u8arr[n] = bstr.charCodeAt(n) + } + resolve(new Blob([u8arr], { type: mime })) + }) +} export const AdamClientState = defineState({ name: 'hexafield.adam-template.AdamClientState', @@ -69,7 +102,126 @@ export const AgentState = defineState({ adam.agent.me().then((response) => { getMutableState(AgentState).set(response) + P2P_API.client = AgentBlobAPI }) }, [adam]) } }) + +export const AgentBlobAPI: CRUD_API = { + create: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + await client.languages.byAddress(FILE_STORAGE_LANGUAGE) + + const data_base64 = await blobToDataURL(args.file) + const target = await client.expression.create( + { + data_base64, + name: args.fileName, + file_type: args.fileType + }, + FILE_STORAGE_LANGUAGE + ) + + const newLink = new Link({ + source: args.source, + predicate: args.predicate, + target + }) + + await client.agent.mutatePublicPerspective({ + additions: [newLink], + removals: [] + } as LinkMutations) + }, + + get: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = getState(AgentState) + if (!myPerspectives) throw new Error('Agent state not initialized') + + const myLinks = myPerspectives.perspective!.links + + const link = myLinks.find((link) => link.data.source === args.source && link.data.predicate === args.predicate) + if (!link) return undefined // not found + + const res = await client.expression.get(link.data.target) + if (!res) return undefined // not found + + const { data } = res + const { data_base64, file_type } = JSON.parse(data) + if (!data_base64) return undefined + const b64 = `data:${file_type};base64, ${data_base64}` + + const blob = await dataURLToBlob(b64) + + return blob + }, + + replace: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = getState(AgentState) + if (!myPerspectives) throw new Error('Agent state not initialized') + + const myLinks = myPerspectives.perspective!.links + + const currentLink = myLinks.find( + (link) => link.data.source === args.source && link.data.predicate === args.predicate + ) + if (!currentLink) throw new Error('Current link not found') + + const removals = [currentLink] + + await client.languages.byAddress(FILE_STORAGE_LANGUAGE) + + const data_base64 = await blobToDataURL(args.file) + const target = await client.expression.create( + { + data_base64, + name: args.fileName, + file_type: args.fileType + }, + FILE_STORAGE_LANGUAGE + ) + + const newLink = new Link({ + source: args.source, + predicate: args.predicate, + target + }) + + await client.agent.mutatePublicPerspective({ + additions: [newLink], + removals: removals + }) + }, + + delete: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = getState(AgentState) + if (!myPerspectives) throw new Error('Agent state not initialized') + + const myLinks = myPerspectives.perspective!.links + + const currentLink = myLinks.find( + (link) => link.data.source === args.source && link.data.predicate === args.predicate + ) + if (!currentLink) throw new Error('Current link not found') + + await client.agent.mutatePublicPerspective({ + additions: [], + removals: [currentLink] + }) + } +} + +globalThis.AdamClientState = AdamClientState +globalThis.AgentState = AgentState +globalThis.AgentBlobAPI = AgentBlobAPI diff --git a/src/api/CRUD.ts b/src/api/CRUD.ts new file mode 100644 index 0000000..0fd80e5 --- /dev/null +++ b/src/api/CRUD.ts @@ -0,0 +1,21 @@ +export type QueryParams = { + source: string + predicate: string +} + +export type FileParams = QueryParams & { + file: Blob + fileName: string + fileType: string +} + +export type CRUD_API = { + create: (args: FileParams) => Promise + get: (args: QueryParams) => Promise + replace: (args: FileParams) => Promise + delete: (args: QueryParams) => Promise +} + +export const P2P_API = { + client: null! as CRUD_API +} From 8d3c7416f4bf130f001c215c27ef1aca9b61e552 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sat, 30 Aug 2025 18:03:53 +1000 Subject: [PATCH 02/22] file sync with adam --- src/ad4m/useADAM.tsx | 41 +++++++++++-- src/api/CRUD.ts | 10 ++- src/api/local.ts | 69 +++++++++++++++++++++ src/tools/GraphPage.tsx | 19 +++++- src/tools/registries/SchemaRegistry.tsx | 81 ++++++++++++++++++++++++- src/tools/registries/constants.ts | 2 + 6 files changed, 211 insertions(+), 11 deletions(-) create mode 100644 src/api/local.ts create mode 100644 src/tools/registries/constants.ts diff --git a/src/ad4m/useADAM.tsx b/src/ad4m/useADAM.tsx index 1e1aa62..6aa6a28 100644 --- a/src/ad4m/useADAM.tsx +++ b/src/ad4m/useADAM.tsx @@ -103,6 +103,7 @@ export const AgentState = defineState({ adam.agent.me().then((response) => { getMutableState(AgentState).set(response) P2P_API.client = AgentBlobAPI + getMutableState(P2P_API).ready.set(true) }) }, [adam]) } @@ -136,12 +137,25 @@ export const AgentBlobAPI: CRUD_API = { } as LinkMutations) }, + has: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') + + const myLinks = myPerspectives.perspective!.links + + const link = myLinks.find((link) => link.data.source === args.source && link.data.predicate === args.predicate) + return !!link + }, + get: async (args) => { const client = getState(AdamClientState) if (!client) throw new Error('AD4M client not initialized') - const myPerspectives = getState(AgentState) - if (!myPerspectives) throw new Error('Agent state not initialized') + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') const myLinks = myPerspectives.perspective!.links @@ -161,12 +175,25 @@ export const AgentBlobAPI: CRUD_API = { return blob }, + find: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') + + const myLinks = myPerspectives.perspective!.links + + const foundLinks = myLinks.filter((link) => link.data.predicate === args.predicate) + return foundLinks.map((link) => link.data.source) + }, + replace: async (args) => { const client = getState(AdamClientState) if (!client) throw new Error('AD4M client not initialized') - const myPerspectives = getState(AgentState) - if (!myPerspectives) throw new Error('Agent state not initialized') + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') const myLinks = myPerspectives.perspective!.links @@ -205,8 +232,8 @@ export const AgentBlobAPI: CRUD_API = { const client = getState(AdamClientState) if (!client) throw new Error('AD4M client not initialized') - const myPerspectives = getState(AgentState) - if (!myPerspectives) throw new Error('Agent state not initialized') + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') const myLinks = myPerspectives.perspective!.links @@ -225,3 +252,5 @@ export const AgentBlobAPI: CRUD_API = { globalThis.AdamClientState = AdamClientState globalThis.AgentState = AgentState globalThis.AgentBlobAPI = AgentBlobAPI + +// wipe data: `AgentBlobAPI.find({ predicate: 'conjure://schema' }).then(sources => sources.forEach(source => AgentBlobAPI.delete({ source, predicate: 'conjure://schema' })))` diff --git a/src/api/CRUD.ts b/src/api/CRUD.ts index 0fd80e5..cc97fd1 100644 --- a/src/api/CRUD.ts +++ b/src/api/CRUD.ts @@ -1,3 +1,5 @@ +import { defineState } from '@ir-engine/hyperflux' + export type QueryParams = { source: string predicate: string @@ -12,10 +14,14 @@ export type FileParams = QueryParams & { export type CRUD_API = { create: (args: FileParams) => Promise get: (args: QueryParams) => Promise + has: (args: QueryParams) => Promise + find: (args: { predicate: string }) => Promise // Find files by predicate, returns list of sources replace: (args: FileParams) => Promise delete: (args: QueryParams) => Promise } -export const P2P_API = { +export const P2P_API = defineState({ + name: 'hexafield.conjure.P2P_API', + initial: { ready: false }, client: null! as CRUD_API -} +}) diff --git a/src/api/local.ts b/src/api/local.ts new file mode 100644 index 0000000..d4508ab --- /dev/null +++ b/src/api/local.ts @@ -0,0 +1,69 @@ +import type { CRUD_API } from './CRUD' + +const url = 'https://localhost:8000' + +export const LocalBlobAPI: CRUD_API = { + create: async (args) => { + const form = new FormData() + form.append('source', args.source) + form.append('predicate', args.predicate) + form.append('fileName', args.fileName) + form.append('fileType', args.fileType) + form.append('file', args.file, args.fileName) + + const response = await fetch(`${url}/create`, { method: 'POST', body: form }) + if (!response.ok) throw new Error('Failed to create blob') + }, + + get: async (args) => { + const response = await fetch(`${url}/get`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (response.status === 404) return undefined + if (!response.ok) throw new Error('Failed to get blob') + return response.blob() + }, + + has: async (args) => { + const response = await fetch(`${url}/has`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (!response.ok) throw new Error('Failed to check blob existence') + return response.json() + }, + + find: async (args) => { + const response = await fetch(`${url}/find`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (!response.ok) throw new Error('Failed to find blobs') + return response.json() + }, + + replace: async (args) => { + const form = new FormData() + form.append('source', args.source) + form.append('predicate', args.predicate) + form.append('fileName', args.fileName) + form.append('fileType', args.fileType) + form.append('file', args.file, args.fileName) + + const response = await fetch(`${url}/replace`, { method: 'POST', body: form }) + if (!response.ok) throw new Error('Failed to replace blob') + }, + + delete: async (args) => { + const response = await fetch(`${url}/delete`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (response.status !== 404 && !response.ok) throw new Error('Failed to delete blob') + } +} diff --git a/src/tools/GraphPage.tsx b/src/tools/GraphPage.tsx index abe7e1d..1e82a54 100644 --- a/src/tools/GraphPage.tsx +++ b/src/tools/GraphPage.tsx @@ -8,7 +8,7 @@ import { Vector3 } from 'three' import Debug from '@ir-engine/client-core/src/components/Debug' import { useDraggable } from '@ir-engine/client-core/src/hooks/useDraggable' import { createEntity, EntityTreeComponent, removeEntity, setComponent } from '@ir-engine/ecs' -import { getMutableState, useHookstate, useMutableState, useReactiveRef } from '@ir-engine/hyperflux' +import { getMutableState, getState, useHookstate, useMutableState, useReactiveRef } from '@ir-engine/hyperflux' import { AmbientLightComponent, ReferenceSpaceState, TransformComponent } from '@ir-engine/spatial' import { CameraOrbitComponent } from '@ir-engine/spatial/src/camera/components/CameraOrbitComponent' import { NameComponent } from '@ir-engine/spatial/src/common/NameComponent' @@ -25,6 +25,7 @@ import ToolView from './views/ToolView' import Tabs from './components/Tabs' +import { P2P_API } from '../api/CRUD' import './graph/forcegraph/ForceGraph' const tabs = [ @@ -139,11 +140,27 @@ function ToolUI() { export default function GraphPage() { const [ref, setRef] = useReactiveRef() + const storageMethod = useHookstate<'local' | 'ADAM'>('ADAM') + useSpatialEngine() useEngineCanvas(ref) const { originEntity, viewerEntity } = useMutableState(ReferenceSpaceState).value + useEffect(() => { + if (storageMethod.value === 'ADAM') { + import('../ad4m/useADAM').then((module) => { + // get agent state to initialize agent + getState(module.AgentState) + }) + } else { + import('../api/local').then((module) => { + P2P_API.client = module.LocalBlobAPI + getMutableState(P2P_API).ready.set(true) + }) + } + }, []) + useEffect(() => { if (!originEntity || !viewerEntity) return diff --git a/src/tools/registries/SchemaRegistry.tsx b/src/tools/registries/SchemaRegistry.tsx index 17095c7..fef4909 100644 --- a/src/tools/registries/SchemaRegistry.tsx +++ b/src/tools/registries/SchemaRegistry.tsx @@ -1,6 +1,16 @@ -import { defineState, getMutableState, none, syncStateWithLocalStorage } from '@ir-engine/hyperflux' +import { + defineState, + getMutableState, + NO_PROXY, + none, + syncStateWithLocalStorage, + useMutableState +} from '@ir-engine/hyperflux' +import React, { useEffect } from 'react' +import { P2P_API } from '../../api/CRUD' import { JSONSchemaType } from '../json-schema/JSONSchema' import { contentHash } from '../json-schema/contentHash' +import { SCHEMA_PREDICATE } from './constants' export type SHA256Hash = string @@ -16,6 +26,7 @@ export const SchemaRegistry = defineState({ initial: { schemas: {} as Record }, register: (schema: JSONSchemaType, label?: string, description?: string) => { + console.log(schema, label, description) const hash = contentHash(schema) getMutableState(SchemaRegistry).schemas[hash].set({ hash, @@ -28,7 +39,73 @@ export const SchemaRegistry = defineState({ forget: (hash: SHA256Hash) => { getMutableState(SchemaRegistry).schemas[hash].set(none) + if (!P2P_API.client) return + P2P_API.client.delete({ source: hash, predicate: SCHEMA_PREDICATE }).then(async () => { + console.log('deleted:', hash) + }) }, - extension: syncStateWithLocalStorage(['schemas']) + extension: syncStateWithLocalStorage(['schemas']), + + reactor: () => { + const schemaState = useMutableState(SchemaRegistry).schemas + const apiReady = useMutableState(P2P_API).ready.value + + useEffect(() => { + if (!apiReady) return + P2P_API.client.find({ predicate: SCHEMA_PREDICATE }).then((sources) => { + console.log(sources) + sources.forEach(async (source) => { + P2P_API.client + .get({ source, predicate: SCHEMA_PREDICATE }) + .then(async (blob) => { + if (!blob) return + console.log({ blob }, await blob.text()) + const { schema, label, description } = JSON.parse(await blob.text()) + SchemaRegistry.register(schema, label, description) + }) + .catch((e) => { + console.error('Failed to retrieve schema:', e) + }) + }) + }) + }, [apiReady]) + + if (!apiReady) return null + + return ( + <> + {schemaState.keys.map((key) => ( + + ))} + + ) + } }) + +const SyncSchema = ({ hash }: { hash: string }) => { + const schema = useMutableState(SchemaRegistry).schemas[hash].get(NO_PROXY) + + useEffect(() => { + P2P_API.client.has({ source: hash, predicate: SCHEMA_PREDICATE }).then(async (exists) => { + console.log('exists:', exists) + if (exists) return + P2P_API.client + .create({ + predicate: SCHEMA_PREDICATE, + source: hash, + file: new Blob( + [JSON.stringify({ schema: schema.schema, label: schema.label, description: schema.description })], + { type: 'application/json' } + ), + fileName: `${hash}.json`, + fileType: 'application/json' + }) + .catch((e) => { + console.error('Failed to create schema:', e) + }) + }) + }, [JSON.stringify(schema)]) + + return null +} diff --git a/src/tools/registries/constants.ts b/src/tools/registries/constants.ts new file mode 100644 index 0000000..3d896d9 --- /dev/null +++ b/src/tools/registries/constants.ts @@ -0,0 +1,2 @@ +export const SCHEMA_PREDICATE = 'conjure://schema' +export const TOOL_PREDICATE = 'conjure://tool' From 750ac59c5d61b2eda3a1b0a8f5bc8596ae6a3385 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sat, 30 Aug 2025 20:16:02 +1000 Subject: [PATCH 03/22] Add file server and update CRUD API for upload management --- .gitignore | 4 +- fileserver.ts | 184 ++++++++++++++++++++++++ package.json | 4 +- src/ad4m/useADAM.tsx | 45 +----- src/api/CRUD.ts | 14 +- src/api/local.ts | 42 +++--- src/tools/registries/SchemaRegistry.tsx | 16 +-- 7 files changed, 227 insertions(+), 82 deletions(-) create mode 100644 fileserver.ts diff --git a/.gitignore b/.gitignore index f149c73..efd1034 100755 --- a/.gitignore +++ b/.gitignore @@ -113,4 +113,6 @@ index.ts.bak public/projects/* -public/data/* \ No newline at end of file +public/data/* + +storage/ \ No newline at end of file diff --git a/fileserver.ts b/fileserver.ts new file mode 100644 index 0000000..99617b7 --- /dev/null +++ b/fileserver.ts @@ -0,0 +1,184 @@ +import express, { Request } from 'express' +import fs from 'fs' +import https from 'https' +import path from 'path' + +type QueryParams = { + source: string + predicate: string +} + +type FileParams = QueryParams & { + target: string +} + +// Basic config +const PORT = 8000 +// Resolve repo-level dev certs; this file lives in packages/projects/projects/hexafield/conjure/ +const CERT_PATH = path.resolve(__dirname, '../../../../../certs/cert.pem') +const KEY_PATH = path.resolve(__dirname, '../../../../../certs/key.pem') + +// Local storage directory for uploaded files +const STORAGE_DIR = path.resolve(__dirname, 'storage') +fs.mkdirSync(STORAGE_DIR, { recursive: true }) + +// Minimal CORS for local dev +const corsMiddleware: express.RequestHandler = (req, res, next) => { + console.log(`[fileserver] ${req.method} ${req.url}`) + res.header('Access-Control-Allow-Origin', req.headers.origin || '*') + res.header('Access-Control-Allow-Credentials', 'true') + res.header('Access-Control-Allow-Methods', 'GET,POST,OPTIONS') + res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization') + if (req.method === 'OPTIONS') return res.sendStatus(204) + next() +} + +const app = express() +app.use(corsMiddleware) +app.use(express.json({ limit: '10mb' })) // JSON only for metadata routes + +const keyFor = ({ source, predicate }: QueryParams) => `${encodeURIComponent(predicate)}__${encodeURIComponent(source)}` + +const pathFor = (q: QueryParams) => path.join(STORAGE_DIR, `${keyFor(q)}.json`) + +const readFile = (q: QueryParams): FileParams | undefined => { + const path = pathFor(q) + if (!fs.existsSync(path)) return undefined + try { + return JSON.parse(fs.readFileSync(path, 'utf-8')) as FileParams + } catch { + return undefined + } +} + +const writeFile = (q: QueryParams, data: string) => { + fs.writeFileSync(pathFor(q), data) +} + +app.get('/health', (_req, res) => res.status(200).send('ok')) + +app.post('/create', (req: Request, res) => { + try { + const { source, predicate, target } = req.body as unknown as FileParams + if (!source || !predicate || !target) { + return res.status(400).json({ error: 'Missing required fields' }) + } + + const q: QueryParams = { source, predicate } + const finalPath = pathFor(q) + + if (finalPath && fs.existsSync(finalPath)) fs.unlinkSync(finalPath) + + writeFile(q, target) + + return res.status(201).json({ ok: true }) + } catch (e) { + console.error('CREATE error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// Get (download) +app.post('/get', (req, res) => { + try { + const { source, predicate } = req.body as QueryParams + if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' }) + + const path = pathFor({ source, predicate }) + if (!path || !fs.existsSync(path)) return res.status(404).json({ error: 'Not found' }) + + const file = readFile({ source, predicate }) + if (!file) return res.status(404).json({ error: 'Not found' }) + + return res.status(200).json(file) + } catch (e) { + console.error('GET error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// Find (return results for matching predicate) +app.post('/find', (req, res) => { + try { + const { predicate } = req.body as { predicate?: string } + if (!predicate) return res.status(400).json({ error: 'Missing required fields' }) + + // Scan storage dir for all .json meta files + const results: Array = [] + const files = fs.readdirSync(STORAGE_DIR) + for (const file of files) { + if (!file.endsWith('.json')) continue + const encodedPredicate = encodeURIComponent(predicate) + if (file.startsWith(encodedPredicate)) { + results.push(decodeURIComponent(file.slice(encodedPredicate.length + 2, -5))) + } + } + return res.status(200).json({ ok: true, results }) + } catch (e) { + console.error('FIND error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +app.post('/has', (req, res) => { + try { + const { source, predicate } = req.body as QueryParams + if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' }) + + const file = fs.existsSync(pathFor({ source, predicate })) + if (!file) return res.status(200).json({ ok: false }) + + return res.status(200).json({ ok: true }) + } catch (e) { + console.error('HAS error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// Replace +app.post('/replace', (req: Request, res) => { + try { + const { source, predicate, target } = req.body as unknown as FileParams + if (!source || !predicate || !target) { + return res.status(400).json({ error: 'Missing required fields' }) + } + const q: QueryParams = { source, predicate } + const path = pathFor(q) + if (!path) { + return res.status(404).json({ error: 'Not found' }) + } + // Remove old file + if (path && fs.existsSync(path)) fs.unlinkSync(path) + + writeFile(q, target) + + return res.status(200).json({ ok: true }) + } catch (e) { + console.error('REPLACE error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// Delete +app.post('/delete', (req, res) => { + try { + const { source, predicate } = req.body as QueryParams + if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' }) + const q: QueryParams = { source, predicate } + + const path = pathFor(q) + if (fs.existsSync(path)) fs.unlinkSync(path) + + return res.status(200).json({ ok: true }) + } catch (e) { + console.error('DELETE error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// HTTPS server +const key = fs.readFileSync(KEY_PATH) +const cert = fs.readFileSync(CERT_PATH) +https.createServer({ key, cert }, app).listen(PORT, () => { + console.log(`[fileserver] HTTPS listening on https://localhost:${PORT}`) +}) diff --git a/package.json b/package.json index f195668..b9d103a 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "build:worker": "esbuild src/tools/graph/forcegraph/ForceGraphWorker.ts --bundle --format=esm --outfile=src/tools/graph/forcegraph/ForceGraphWorker.bundle.js --loader:.wasm=binary", "build:dag-worker": "esbuild src/tools/graph/dag/DagWorker.ts --bundle --format=esm --outfile=src/tools/graph/dag/DagWorker.bundle.js --loader:.wasm=binary", "start": "tsx server.js", + "fileserver": "tsx fileserver.ts", "format": "prettier --write \"**/*.{ts,tsx}\"", "format-scss": "stylelint \"**/*.scss\" --fix", "format-staged": "lint-staged", @@ -31,7 +32,8 @@ "jsonpath-plus": "10.3.0", "maxrects-packer": "2.7.3", "monaco-editor": "^0.52.2", - "ts-morph": "^26.0.0" + "ts-morph": "^26.0.0", + "express": "^4.19.2" }, "license": "ISC", "pre-commit": [ diff --git a/src/ad4m/useADAM.tsx b/src/ad4m/useADAM.tsx index 6aa6a28..b6da9fc 100644 --- a/src/ad4m/useADAM.tsx +++ b/src/ad4m/useADAM.tsx @@ -1,13 +1,9 @@ -import { Ad4mClient, Agent, Link, LinkMutations } from '@coasys/ad4m' +import { Ad4mClient, Agent, Link, LinkMutations, Literal } from '@coasys/ad4m' import Ad4mConnectUI, { Ad4mConnectElement, getAd4mClient } from '@coasys/ad4m-connect' import { defineState, getMutableState, getState, useHookstate, useMutableState } from '@ir-engine/hyperflux' import { useEffect } from 'react' import { CRUD_API, P2P_API } from '../api/CRUD' -// import { languages } from '@coasys/flux-constants' -const languages = { FILE_STORAGE_LANGUAGE: 'QmzSYwdjqeP9D13Sfmyc5HcabM9jL3DtPyhadnF6dQXu4FjVSbQ' } -const { FILE_STORAGE_LANGUAGE } = languages - export const blobToDataURL = (blob: Blob): Promise => { return new Promise((resolve, reject) => { const reader = new FileReader() @@ -113,22 +109,11 @@ export const AgentBlobAPI: CRUD_API = { create: async (args) => { const client = getState(AdamClientState) if (!client) throw new Error('AD4M client not initialized') - await client.languages.byAddress(FILE_STORAGE_LANGUAGE) - - const data_base64 = await blobToDataURL(args.file) - const target = await client.expression.create( - { - data_base64, - name: args.fileName, - file_type: args.fileType - }, - FILE_STORAGE_LANGUAGE - ) const newLink = new Link({ source: args.source, predicate: args.predicate, - target + target: Literal.from(args.target).toUrl() }) await client.agent.mutatePublicPerspective({ @@ -162,17 +147,7 @@ export const AgentBlobAPI: CRUD_API = { const link = myLinks.find((link) => link.data.source === args.source && link.data.predicate === args.predicate) if (!link) return undefined // not found - const res = await client.expression.get(link.data.target) - if (!res) return undefined // not found - - const { data } = res - const { data_base64, file_type } = JSON.parse(data) - if (!data_base64) return undefined - const b64 = `data:${file_type};base64, ${data_base64}` - - const blob = await dataURLToBlob(b64) - - return blob + return Literal.fromUrl(link.data.target).get() }, find: async (args) => { @@ -204,22 +179,10 @@ export const AgentBlobAPI: CRUD_API = { const removals = [currentLink] - await client.languages.byAddress(FILE_STORAGE_LANGUAGE) - - const data_base64 = await blobToDataURL(args.file) - const target = await client.expression.create( - { - data_base64, - name: args.fileName, - file_type: args.fileType - }, - FILE_STORAGE_LANGUAGE - ) - const newLink = new Link({ source: args.source, predicate: args.predicate, - target + target: Literal.from(args.target).toUrl() }) await client.agent.mutatePublicPerspective({ diff --git a/src/api/CRUD.ts b/src/api/CRUD.ts index cc97fd1..e73d7be 100644 --- a/src/api/CRUD.ts +++ b/src/api/CRUD.ts @@ -5,18 +5,18 @@ export type QueryParams = { predicate: string } -export type FileParams = QueryParams & { - file: Blob - fileName: string - fileType: string +export type RDFParams = QueryParams & { + target: TargetType } +export type TargetType = string | object | boolean | number + export type CRUD_API = { - create: (args: FileParams) => Promise - get: (args: QueryParams) => Promise + create: (args: RDFParams) => Promise + get: (args: QueryParams) => Promise has: (args: QueryParams) => Promise find: (args: { predicate: string }) => Promise // Find files by predicate, returns list of sources - replace: (args: FileParams) => Promise + replace: (args: RDFParams) => Promise delete: (args: QueryParams) => Promise } diff --git a/src/api/local.ts b/src/api/local.ts index d4508ab..9b4878f 100644 --- a/src/api/local.ts +++ b/src/api/local.ts @@ -4,14 +4,15 @@ const url = 'https://localhost:8000' export const LocalBlobAPI: CRUD_API = { create: async (args) => { - const form = new FormData() - form.append('source', args.source) - form.append('predicate', args.predicate) - form.append('fileName', args.fileName) - form.append('fileType', args.fileType) - form.append('file', args.file, args.fileName) - - const response = await fetch(`${url}/create`, { method: 'POST', body: form }) + const response = await fetch(`${url}/create`, { + method: 'POST', + body: JSON.stringify({ + source: args.source, + predicate: args.predicate, + target: JSON.stringify(args.target) + }), + headers: { 'Content-Type': 'application/json' } + }) if (!response.ok) throw new Error('Failed to create blob') }, @@ -23,7 +24,8 @@ export const LocalBlobAPI: CRUD_API = { }) if (response.status === 404) return undefined if (!response.ok) throw new Error('Failed to get blob') - return response.blob() + const json = await response.json() + return json.target }, has: async (args) => { @@ -32,8 +34,7 @@ export const LocalBlobAPI: CRUD_API = { body: JSON.stringify(args), headers: { 'Content-Type': 'application/json' } }) - if (!response.ok) throw new Error('Failed to check blob existence') - return response.json() + return (await response.json()).ok }, find: async (args) => { @@ -43,18 +44,19 @@ export const LocalBlobAPI: CRUD_API = { headers: { 'Content-Type': 'application/json' } }) if (!response.ok) throw new Error('Failed to find blobs') - return response.json() + return (await response.json()).results }, replace: async (args) => { - const form = new FormData() - form.append('source', args.source) - form.append('predicate', args.predicate) - form.append('fileName', args.fileName) - form.append('fileType', args.fileType) - form.append('file', args.file, args.fileName) - - const response = await fetch(`${url}/replace`, { method: 'POST', body: form }) + const response = await fetch(`${url}/replace`, { + method: 'POST', + body: JSON.stringify({ + source: args.source, + predicate: args.predicate, + target: JSON.stringify(args.target) + }), + headers: { 'Content-Type': 'application/json' } + }) if (!response.ok) throw new Error('Failed to replace blob') }, diff --git a/src/tools/registries/SchemaRegistry.tsx b/src/tools/registries/SchemaRegistry.tsx index fef4909..704a385 100644 --- a/src/tools/registries/SchemaRegistry.tsx +++ b/src/tools/registries/SchemaRegistry.tsx @@ -54,14 +54,12 @@ export const SchemaRegistry = defineState({ useEffect(() => { if (!apiReady) return P2P_API.client.find({ predicate: SCHEMA_PREDICATE }).then((sources) => { - console.log(sources) sources.forEach(async (source) => { P2P_API.client .get({ source, predicate: SCHEMA_PREDICATE }) - .then(async (blob) => { - if (!blob) return - console.log({ blob }, await blob.text()) - const { schema, label, description } = JSON.parse(await blob.text()) + .then(async (response: object) => { + if (!response) return + const { schema, label, description } = response as any SchemaRegistry.register(schema, label, description) }) .catch((e) => { @@ -88,18 +86,12 @@ const SyncSchema = ({ hash }: { hash: string }) => { useEffect(() => { P2P_API.client.has({ source: hash, predicate: SCHEMA_PREDICATE }).then(async (exists) => { - console.log('exists:', exists) if (exists) return P2P_API.client .create({ predicate: SCHEMA_PREDICATE, source: hash, - file: new Blob( - [JSON.stringify({ schema: schema.schema, label: schema.label, description: schema.description })], - { type: 'application/json' } - ), - fileName: `${hash}.json`, - fileType: 'application/json' + target: { schema: schema.schema, label: schema.label, description: schema.description } }) .catch((e) => { console.error('Failed to create schema:', e) From 9d1d9a0ded6223e8ed538c6634d95dc362d70da6 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sat, 30 Aug 2025 20:24:40 +1000 Subject: [PATCH 04/22] Add p2p storage to tool registry --- src/tools/registries/ToolRegistry.tsx | 91 +++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/src/tools/registries/ToolRegistry.tsx b/src/tools/registries/ToolRegistry.tsx index 2490ebd..0760f5e 100644 --- a/src/tools/registries/ToolRegistry.tsx +++ b/src/tools/registries/ToolRegistry.tsx @@ -3,16 +3,19 @@ import { defineState, getMutableState, getState, + NO_PROXY, none, syncStateWithLocalStorage, - useHookstate + useMutableState } from '@ir-engine/hyperflux' -import { useEffect } from 'react' +import React, { useEffect } from 'react' +import { P2P_API } from '../../api/CRUD' import { JSONSchemaType } from '../json-schema/JSONSchema' import { contentHash } from '../json-schema/contentHash' import { createDynamicWebworker } from '../utils/createDynamicWebworker' import { hashFunctionSource } from '../utils/hashFunction' import { SchemaRegistry, SHA256Hash } from './SchemaRegistry' +import { TOOL_PREDICATE } from './constants' export type Stringify = string & { __fnSignature: Signature @@ -70,6 +73,11 @@ export const ToolRegistry = defineState({ forget: (hash: SHA256Hash) => { getMutableState(ToolRegistry).tools[hash].set(none) + + if (!P2P_API.client) return + P2P_API.client.delete({ source: hash, predicate: TOOL_PREDICATE }).then(async () => { + console.log('deleted:', hash) + }) }, run: async (hash: SHA256Hash, input: Input): Promise => { @@ -105,18 +113,79 @@ export const ToolRegistry = defineState({ extension: syncStateWithLocalStorage(['tools']), reactor: () => { - const state = useHookstate(getMutableState(ToolRegistry).tools) + const toolState = useMutableState(ToolRegistry).tools + const apiReady = useMutableState(P2P_API).ready.value useEffect(() => { - const tools = getState(ToolRegistry).tools - for (const [hash, tool] of Object.entries(tools)) { - if (!getState(SchemaRegistry).schemas[tool.inputHash]) - SchemaRegistry.register(tool.input, tool.label, tool.description) - if (!getState(SchemaRegistry).schemas[tool.outputHash]) - SchemaRegistry.register(tool.output, tool.label, tool.description) - } - }, [state]) + if (!apiReady) return + P2P_API.client.find({ predicate: TOOL_PREDICATE }).then((sources) => { + sources.forEach(async (source) => { + P2P_API.client + .get({ source, predicate: TOOL_PREDICATE }) + .then(async (response: object) => { + if (!response) return + const { label, description, input, output, transformation } = response as Tool + ToolRegistry.create({ + label, + description, + input, + output, + transformation + }) + }) + .catch((e) => { + console.error('Failed to retrieve schema:', e) + }) + }) + }) + }, [apiReady]) + + if (!apiReady) return null + + return ( + <> + {toolState.keys.map((key) => ( + + ))} + + ) } }) +const SyncTool = ({ hash }: { hash: string }) => { + const tool = useMutableState(ToolRegistry).tools[hash].get(NO_PROXY) + + useEffect(() => { + if (!getState(SchemaRegistry).schemas[tool.inputHash]) + SchemaRegistry.register(tool.input, tool.label, tool.description) + if (!getState(SchemaRegistry).schemas[tool.outputHash]) + SchemaRegistry.register(tool.output, tool.label, tool.description) + + P2P_API.client.has({ source: hash, predicate: TOOL_PREDICATE }).then(async (exists) => { + if (exists) return + P2P_API.client + .create({ + predicate: TOOL_PREDICATE, + source: hash, + target: { + hash: tool.hash, + label: tool.label, + description: tool.description, + input: tool.input, + output: tool.output, + transformation: tool.transformation, + inputHash: tool.inputHash, + outputHash: tool.outputHash, + transformationHash: tool.transformationHash + } + }) + .catch((e) => { + console.error('Failed to create tool:', e) + }) + }) + }, [JSON.stringify(tool)]) + + return null +} + const workers = {} as Record>> From 475fe4e3cdbe656008b04945af3ec3ad2787337a Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sat, 30 Aug 2025 20:37:15 +1000 Subject: [PATCH 05/22] Add local storage API --- src/api/local.ts | 102 ++++++++++++++++++++-------------------- src/api/server.ts | 71 ++++++++++++++++++++++++++++ src/tools/GraphPage.tsx | 11 +++-- 3 files changed, 131 insertions(+), 53 deletions(-) create mode 100644 src/api/server.ts diff --git a/src/api/local.ts b/src/api/local.ts index 9b4878f..a6bfadb 100644 --- a/src/api/local.ts +++ b/src/api/local.ts @@ -1,71 +1,73 @@ -import type { CRUD_API } from './CRUD' +import type { CRUD_API, QueryParams } from './CRUD' -const url = 'https://localhost:8000' +const conjureCacheKey = 'CONJURE_STORAGE_CACHE' + +const localCache = { entries: {} as Record, predicateIndex: {} as Record } + +const saveCache = () => { + localStorage.setItem(conjureCacheKey, JSON.stringify(localCache.entries)) +} + +const loadCache = () => { + const cached = localStorage.getItem(conjureCacheKey) + if (cached) { + Object.assign(localCache.entries, JSON.parse(cached)) + } + localCache.predicateIndex = Object.keys(localCache.entries).reduce( + (acc, key) => { + const [predicate] = key.split('__') + if (!acc[predicate]) { + acc[predicate] = [] + } + acc[predicate].push(key) + return acc + }, + {} as Record + ) +} + +loadCache() + +const keyFor = ({ source, predicate }: QueryParams) => `${encodeURIComponent(predicate)}__${encodeURIComponent(source)}` export const LocalBlobAPI: CRUD_API = { create: async (args) => { - const response = await fetch(`${url}/create`, { - method: 'POST', - body: JSON.stringify({ - source: args.source, - predicate: args.predicate, - target: JSON.stringify(args.target) - }), - headers: { 'Content-Type': 'application/json' } - }) - if (!response.ok) throw new Error('Failed to create blob') + const key = keyFor(args) + localCache.entries[key] = JSON.stringify(args.target) + saveCache() }, get: async (args) => { - const response = await fetch(`${url}/get`, { - method: 'POST', - body: JSON.stringify(args), - headers: { 'Content-Type': 'application/json' } - }) - if (response.status === 404) return undefined - if (!response.ok) throw new Error('Failed to get blob') - const json = await response.json() - return json.target + const key = keyFor(args) + const value = localCache.entries[key] + return value ? JSON.parse(value) : undefined }, has: async (args) => { - const response = await fetch(`${url}/has`, { - method: 'POST', - body: JSON.stringify(args), - headers: { 'Content-Type': 'application/json' } - }) - return (await response.json()).ok + const key = keyFor(args) + return localCache.entries[key] !== undefined }, find: async (args) => { - const response = await fetch(`${url}/find`, { - method: 'POST', - body: JSON.stringify(args), - headers: { 'Content-Type': 'application/json' } - }) - if (!response.ok) throw new Error('Failed to find blobs') - return (await response.json()).results + const results = [] as string[] + for (const key of localCache.predicateIndex[args.predicate] || []) { + const value = localCache.entries[key] + if (typeof value === 'string') { + results.push(JSON.parse(value)) + } + } + return results }, replace: async (args) => { - const response = await fetch(`${url}/replace`, { - method: 'POST', - body: JSON.stringify({ - source: args.source, - predicate: args.predicate, - target: JSON.stringify(args.target) - }), - headers: { 'Content-Type': 'application/json' } - }) - if (!response.ok) throw new Error('Failed to replace blob') + const key = keyFor(args) + localCache.entries[key] = JSON.stringify(args.target) + saveCache() }, delete: async (args) => { - const response = await fetch(`${url}/delete`, { - method: 'POST', - body: JSON.stringify(args), - headers: { 'Content-Type': 'application/json' } - }) - if (response.status !== 404 && !response.ok) throw new Error('Failed to delete blob') + const key = keyFor(args) + delete localCache.entries[key] + saveCache() } } diff --git a/src/api/server.ts b/src/api/server.ts new file mode 100644 index 0000000..10cc987 --- /dev/null +++ b/src/api/server.ts @@ -0,0 +1,71 @@ +import type { CRUD_API } from './CRUD' + +const url = 'https://localhost:8000' + +export const ServerBlobAPI: CRUD_API = { + create: async (args) => { + const response = await fetch(`${url}/create`, { + method: 'POST', + body: JSON.stringify({ + source: args.source, + predicate: args.predicate, + target: JSON.stringify(args.target) + }), + headers: { 'Content-Type': 'application/json' } + }) + if (!response.ok) throw new Error('Failed to create blob') + }, + + get: async (args) => { + const response = await fetch(`${url}/get`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (response.status === 404) return undefined + if (!response.ok) throw new Error('Failed to get blob') + const json = await response.json() + return json.target + }, + + has: async (args) => { + const response = await fetch(`${url}/has`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + return (await response.json()).ok + }, + + find: async (args) => { + const response = await fetch(`${url}/find`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (!response.ok) throw new Error('Failed to find blobs') + return (await response.json()).results + }, + + replace: async (args) => { + const response = await fetch(`${url}/replace`, { + method: 'POST', + body: JSON.stringify({ + source: args.source, + predicate: args.predicate, + target: JSON.stringify(args.target) + }), + headers: { 'Content-Type': 'application/json' } + }) + if (!response.ok) throw new Error('Failed to replace blob') + }, + + delete: async (args) => { + const response = await fetch(`${url}/delete`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (response.status !== 404 && !response.ok) throw new Error('Failed to delete blob') + } +} diff --git a/src/tools/GraphPage.tsx b/src/tools/GraphPage.tsx index 1e82a54..6a167b9 100644 --- a/src/tools/GraphPage.tsx +++ b/src/tools/GraphPage.tsx @@ -140,7 +140,7 @@ function ToolUI() { export default function GraphPage() { const [ref, setRef] = useReactiveRef() - const storageMethod = useHookstate<'local' | 'ADAM'>('ADAM') + const storageMethod = useHookstate<'local' | 'server' | 'adam'>('local') useSpatialEngine() useEngineCanvas(ref) @@ -148,12 +148,17 @@ export default function GraphPage() { const { originEntity, viewerEntity } = useMutableState(ReferenceSpaceState).value useEffect(() => { - if (storageMethod.value === 'ADAM') { + if (storageMethod.value === 'adam') { import('../ad4m/useADAM').then((module) => { // get agent state to initialize agent getState(module.AgentState) }) - } else { + } else if (storageMethod.value === 'server') { + import('../api/server').then((module) => { + P2P_API.client = module.ServerBlobAPI + getMutableState(P2P_API).ready.set(true) + }) + } else if (storageMethod.value === 'local') { import('../api/local').then((module) => { P2P_API.client = module.LocalBlobAPI getMutableState(P2P_API).ready.set(true) From 490d2b70d484eaf3d807f91dd5eeb31d41e2baf9 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sat, 30 Aug 2025 20:44:21 +1000 Subject: [PATCH 06/22] Add dropdown for storage mode --- src/ad4m/useADAM.tsx | 1 + src/tools/GraphPage.tsx | 68 ++++++++++++++++++++++++----------------- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/ad4m/useADAM.tsx b/src/ad4m/useADAM.tsx index b6da9fc..ec2396c 100644 --- a/src/ad4m/useADAM.tsx +++ b/src/ad4m/useADAM.tsx @@ -100,6 +100,7 @@ export const AgentState = defineState({ getMutableState(AgentState).set(response) P2P_API.client = AgentBlobAPI getMutableState(P2P_API).ready.set(true) + console.log('ADAM storage method ready') }) }, [adam]) } diff --git a/src/tools/GraphPage.tsx b/src/tools/GraphPage.tsx index 6a167b9..244a112 100644 --- a/src/tools/GraphPage.tsx +++ b/src/tools/GraphPage.tsx @@ -50,7 +50,36 @@ function ToolMenus(): JSX.Element { } function ToolUI() { + const storageMethod = useHookstate<'local' | 'server' | 'adam'>('local') + const showMappingUI = useHookstate(true) + + useEffect(() => { + if (storageMethod.value === 'adam') { + import('../ad4m/useADAM').then((module) => { + getMutableState(P2P_API).ready.set(false) + P2P_API.client = module.AgentBlobAPI + // get agent state to initialize agent + if (getState(module.AgentState)) { + getMutableState(P2P_API).ready.set(true) + console.log('ADAM storage method ready') + } + }) + } else if (storageMethod.value === 'server') { + import('../api/server').then((module) => { + P2P_API.client = module.ServerBlobAPI + getMutableState(P2P_API).ready.set(true) + console.log('Server storage method ready') + }) + } else if (storageMethod.value === 'local') { + import('../api/local').then((module) => { + P2P_API.client = module.LocalBlobAPI + getMutableState(P2P_API).ready.set(true) + console.log('Local storage method ready') + }) + } + }, [storageMethod.value]) + const size = useHookstate<{ width: number; height: number }>(() => { const w = Number.parseInt(localStorage.getItem('toolUIWidth') || '') const h = Number.parseInt(localStorage.getItem('toolUIHeight') || '') @@ -120,13 +149,7 @@ function ToolUI() { )}

Tool Menu

- {showMappingUI.value && ( -
- -
- )} + @@ -140,32 +163,11 @@ function ToolUI() { export default function GraphPage() { const [ref, setRef] = useReactiveRef() - const storageMethod = useHookstate<'local' | 'server' | 'adam'>('local') - useSpatialEngine() useEngineCanvas(ref) const { originEntity, viewerEntity } = useMutableState(ReferenceSpaceState).value - useEffect(() => { - if (storageMethod.value === 'adam') { - import('../ad4m/useADAM').then((module) => { - // get agent state to initialize agent - getState(module.AgentState) - }) - } else if (storageMethod.value === 'server') { - import('../api/server').then((module) => { - P2P_API.client = module.ServerBlobAPI - getMutableState(P2P_API).ready.set(true) - }) - } else if (storageMethod.value === 'local') { - import('../api/local').then((module) => { - P2P_API.client = module.LocalBlobAPI - getMutableState(P2P_API).ready.set(true) - }) - } - }, []) - useEffect(() => { if (!originEntity || !viewerEntity) return @@ -198,3 +200,13 @@ export default function GraphPage() { ) } + +const StorageDropdown = ({ storageMethod, onChange }) => { + return ( + + ) +} From 1e1b46daec36701339d288f05ec76ffc8c2bc644 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sat, 30 Aug 2025 20:57:30 +1000 Subject: [PATCH 07/22] bug fixes --- src/api/CRUD.ts | 7 ++++--- src/api/local.ts | 11 ++++++++--- src/tools/GraphPage.tsx | 3 +-- src/tools/registries/SchemaRegistry.tsx | 12 +----------- src/tools/registries/ToolRegistry.tsx | 12 +----------- 5 files changed, 15 insertions(+), 30 deletions(-) diff --git a/src/api/CRUD.ts b/src/api/CRUD.ts index e73d7be..24127da 100644 --- a/src/api/CRUD.ts +++ b/src/api/CRUD.ts @@ -1,4 +1,4 @@ -import { defineState } from '@ir-engine/hyperflux' +import { defineState, syncStateWithLocalStorage } from '@ir-engine/hyperflux' export type QueryParams = { source: string @@ -22,6 +22,7 @@ export type CRUD_API = { export const P2P_API = defineState({ name: 'hexafield.conjure.P2P_API', - initial: { ready: false }, - client: null! as CRUD_API + initial: { ready: false, selected: 'local' as 'local' | 'server' | 'adam' }, + client: null! as CRUD_API, + extension: syncStateWithLocalStorage(['selected']) }) diff --git a/src/api/local.ts b/src/api/local.ts index a6bfadb..d3b9ac2 100644 --- a/src/api/local.ts +++ b/src/api/local.ts @@ -3,6 +3,7 @@ import type { CRUD_API, QueryParams } from './CRUD' const conjureCacheKey = 'CONJURE_STORAGE_CACHE' const localCache = { entries: {} as Record, predicateIndex: {} as Record } +globalThis.localCache = localCache const saveCache = () => { localStorage.setItem(conjureCacheKey, JSON.stringify(localCache.entries)) @@ -11,7 +12,10 @@ const saveCache = () => { const loadCache = () => { const cached = localStorage.getItem(conjureCacheKey) if (cached) { - Object.assign(localCache.entries, JSON.parse(cached)) + const parsed = JSON.parse(cached) + for (const key in parsed) { + localCache.entries[key] = parsed[key] + } } localCache.predicateIndex = Object.keys(localCache.entries).reduce( (acc, key) => { @@ -49,11 +53,12 @@ export const LocalBlobAPI: CRUD_API = { }, find: async (args) => { + const encodedPredicate = encodeURIComponent(args.predicate) const results = [] as string[] - for (const key of localCache.predicateIndex[args.predicate] || []) { + for (const key of localCache.predicateIndex[encodedPredicate] || []) { const value = localCache.entries[key] if (typeof value === 'string') { - results.push(JSON.parse(value)) + results.push(decodeURIComponent(key.slice(encodedPredicate.length + 2))) } } return results diff --git a/src/tools/GraphPage.tsx b/src/tools/GraphPage.tsx index 244a112..f1960e0 100644 --- a/src/tools/GraphPage.tsx +++ b/src/tools/GraphPage.tsx @@ -50,8 +50,7 @@ function ToolMenus(): JSX.Element { } function ToolUI() { - const storageMethod = useHookstate<'local' | 'server' | 'adam'>('local') - + const storageMethod = useHookstate(getMutableState(P2P_API).selected) const showMappingUI = useHookstate(true) useEffect(() => { diff --git a/src/tools/registries/SchemaRegistry.tsx b/src/tools/registries/SchemaRegistry.tsx index 704a385..6755796 100644 --- a/src/tools/registries/SchemaRegistry.tsx +++ b/src/tools/registries/SchemaRegistry.tsx @@ -1,11 +1,4 @@ -import { - defineState, - getMutableState, - NO_PROXY, - none, - syncStateWithLocalStorage, - useMutableState -} from '@ir-engine/hyperflux' +import { defineState, getMutableState, NO_PROXY, none, useMutableState } from '@ir-engine/hyperflux' import React, { useEffect } from 'react' import { P2P_API } from '../../api/CRUD' import { JSONSchemaType } from '../json-schema/JSONSchema' @@ -26,7 +19,6 @@ export const SchemaRegistry = defineState({ initial: { schemas: {} as Record }, register: (schema: JSONSchemaType, label?: string, description?: string) => { - console.log(schema, label, description) const hash = contentHash(schema) getMutableState(SchemaRegistry).schemas[hash].set({ hash, @@ -45,8 +37,6 @@ export const SchemaRegistry = defineState({ }) }, - extension: syncStateWithLocalStorage(['schemas']), - reactor: () => { const schemaState = useMutableState(SchemaRegistry).schemas const apiReady = useMutableState(P2P_API).ready.value diff --git a/src/tools/registries/ToolRegistry.tsx b/src/tools/registries/ToolRegistry.tsx index 0760f5e..dc5ddc8 100644 --- a/src/tools/registries/ToolRegistry.tsx +++ b/src/tools/registries/ToolRegistry.tsx @@ -1,13 +1,5 @@ import transform from '@hexafield/jsonpath-object-transform' -import { - defineState, - getMutableState, - getState, - NO_PROXY, - none, - syncStateWithLocalStorage, - useMutableState -} from '@ir-engine/hyperflux' +import { defineState, getMutableState, getState, NO_PROXY, none, useMutableState } from '@ir-engine/hyperflux' import React, { useEffect } from 'react' import { P2P_API } from '../../api/CRUD' import { JSONSchemaType } from '../json-schema/JSONSchema' @@ -110,8 +102,6 @@ export const ToolRegistry = defineState({ } }, - extension: syncStateWithLocalStorage(['tools']), - reactor: () => { const toolState = useMutableState(ToolRegistry).tools const apiReady = useMutableState(P2P_API).ready.value From 5fa7a564328bc78b5554dec1bdd8c700c85e72d7 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sat, 30 Aug 2025 21:17:10 +1000 Subject: [PATCH 08/22] make tsc happy --- src/tools/registries/ToolRegistry.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/registries/ToolRegistry.tsx b/src/tools/registries/ToolRegistry.tsx index dc5ddc8..90cf014 100644 --- a/src/tools/registries/ToolRegistry.tsx +++ b/src/tools/registries/ToolRegistry.tsx @@ -147,9 +147,9 @@ const SyncTool = ({ hash }: { hash: string }) => { useEffect(() => { if (!getState(SchemaRegistry).schemas[tool.inputHash]) - SchemaRegistry.register(tool.input, tool.label, tool.description) + SchemaRegistry.register(tool.input as JSONSchemaType, tool.label, tool.description) if (!getState(SchemaRegistry).schemas[tool.outputHash]) - SchemaRegistry.register(tool.output, tool.label, tool.description) + SchemaRegistry.register(tool.output as JSONSchemaType, tool.label, tool.description) P2P_API.client.has({ source: hash, predicate: TOOL_PREDICATE }).then(async (exists) => { if (exists) return From c100ecac4c939c71fbf23007eaa0b5cebce41c47 Mon Sep 17 00:00:00 2001 From: HexaField <10372036+HexaField@users.noreply.github.com> Date: Sun, 31 Aug 2025 20:25:21 +1000 Subject: [PATCH 09/22] Add pipeline editor and registry --- package.json | 6 +- src/tools/components/PipelineCard.tsx | 50 +++++++ src/tools/new/components/PipelineEditor.tsx | 133 ++++++++++++++++++ src/tools/new/components/app/AppBar.tsx | 32 +++++ src/tools/new/components/app/AppShell.tsx | 23 +++ src/tools/new/components/app/BottomPanel.tsx | 31 ++++ src/tools/new/components/app/LeftSidebar.tsx | 37 +++++ src/tools/new/components/app/RightPane.tsx | 48 +++++++ src/tools/new/components/forms/SchemaForm.tsx | 86 +++++++++++ .../new/components/graph/EditorContext.tsx | 13 ++ .../new/components/graph/GraphWorkbench.tsx | 56 ++++++++ src/tools/new/components/graph/Node.tsx | 92 ++++++++++++ src/tools/new/logic/runGraph.ts | 96 +++++++++++++ src/tools/new/logic/useGraphActions.ts | 22 +++ src/tools/new/state/graphState.ts | 107 ++++++++++++++ src/tools/new/state/libraryState.ts | 58 ++++++++ src/tools/new/state/outputState.ts | 23 +++ src/tools/new/state/selectionState.ts | 23 +++ src/tools/registries/PipelineRegistry.tsx | 101 +++++++++++++ src/tools/registries/SchemaRegistry.tsx | 3 +- src/tools/registries/ToolRegistry.tsx | 3 +- src/tools/registries/constants.ts | 2 - src/tools/views/PipelineView.tsx | 119 ++++++++++++++-- 23 files changed, 1148 insertions(+), 16 deletions(-) create mode 100644 src/tools/components/PipelineCard.tsx create mode 100644 src/tools/new/components/PipelineEditor.tsx create mode 100644 src/tools/new/components/app/AppBar.tsx create mode 100644 src/tools/new/components/app/AppShell.tsx create mode 100644 src/tools/new/components/app/BottomPanel.tsx create mode 100644 src/tools/new/components/app/LeftSidebar.tsx create mode 100644 src/tools/new/components/app/RightPane.tsx create mode 100644 src/tools/new/components/forms/SchemaForm.tsx create mode 100644 src/tools/new/components/graph/EditorContext.tsx create mode 100644 src/tools/new/components/graph/GraphWorkbench.tsx create mode 100644 src/tools/new/components/graph/Node.tsx create mode 100644 src/tools/new/logic/runGraph.ts create mode 100644 src/tools/new/logic/useGraphActions.ts create mode 100644 src/tools/new/state/graphState.ts create mode 100644 src/tools/new/state/libraryState.ts create mode 100644 src/tools/new/state/outputState.ts create mode 100644 src/tools/new/state/selectionState.ts create mode 100644 src/tools/registries/PipelineRegistry.tsx delete mode 100644 src/tools/registries/constants.ts diff --git a/package.json b/package.json index b9d103a..1dcf4e8 100644 --- a/package.json +++ b/package.json @@ -28,12 +28,14 @@ "d3": "7.9.0", "d3-force-3d": "^3.0.5", "dat.gui": "^0.7.9", + "express": "^4.19.2", "json-schema-to-ts": "^3.1.1", "jsonpath-plus": "10.3.0", "maxrects-packer": "2.7.3", "monaco-editor": "^0.52.2", - "ts-morph": "^26.0.0", - "express": "^4.19.2" + "react-rnd": "^10.5.2", + "reactflow": "^11.10.0", + "ts-morph": "^26.0.0" }, "license": "ISC", "pre-commit": [ diff --git a/src/tools/components/PipelineCard.tsx b/src/tools/components/PipelineCard.tsx new file mode 100644 index 0000000..47fc718 --- /dev/null +++ b/src/tools/components/PipelineCard.tsx @@ -0,0 +1,50 @@ +import React, { useMemo, useState } from 'react' +import { PipelineEditor, PipelineGraph } from '../new/components/PipelineEditor' +import type { Pipeline } from '../registries/PipelineRegistry' + +type ToolLite = { hash: string; label: string } + +type Props = { + pipeline: Pipeline + tools: ToolLite[] + onRun: () => void + onSaveGraph: (graph: PipelineGraph, pipeline: Pipeline) => void +} + +export const PipelineCard: React.FC = ({ pipeline, tools, onRun, onSaveGraph }) => { + const [open, setOpen] = useState(false) + const initialGraph = useMemo(() => { + return { + nodes: [...(pipeline.graph?.nodes || [])], + edges: [...(pipeline.graph?.edges || [])] + } + }, [pipeline.hash]) + const [working, setWorking] = useState(initialGraph) + + const save = (g: PipelineGraph) => { + onSaveGraph(g, pipeline) + } + + return ( +
+
+
+ {pipeline.label} + {pipeline.hash.slice(0, 8)} +
+
+ +
+
+ {open && ( +
+ +
+ )} +
+ ) +} + +export default PipelineCard diff --git a/src/tools/new/components/PipelineEditor.tsx b/src/tools/new/components/PipelineEditor.tsx new file mode 100644 index 0000000..fa5684d --- /dev/null +++ b/src/tools/new/components/PipelineEditor.tsx @@ -0,0 +1,133 @@ +import React, { useCallback, useMemo, useState } from 'react' +import { Rnd } from 'react-rnd' +import ReactFlow, { + Background, + Controls, + MiniMap, + Connection, + Edge, + Node, + applyNodeChanges, + applyEdgeChanges, + NodeChange, + EdgeChange +} from 'reactflow' +import 'reactflow/dist/style.css' +import { DbNode } from './graph/Node' +import { EditorContext, ToolLite } from './graph/EditorContext' + +export type PipelineGraph = { nodes: Node[]; edges: Edge[] } + +type Props = { + graph: PipelineGraph + onChange: (graph: PipelineGraph) => void + onRun: () => void + onSave: (graph: PipelineGraph) => void + tools?: ToolLite[] +} + +export const PipelineEditor: React.FC = ({ graph, onChange, onRun, onSave, tools = [] }) => { + const [showLeft, setShowLeft] = useState(true) + const [showRight, setShowRight] = useState(true) + const [showBottom, setShowBottom] = useState(true) + + const nodeTypes = useMemo(() => ({ db: DbNode }), []) + + const onNodesChange = useCallback( + (changes: NodeChange[]) => { + onChange({ nodes: applyNodeChanges(changes, graph.nodes), edges: graph.edges }) + }, + [graph.nodes, graph.edges, onChange] + ) + const onEdgesChange = useCallback( + (changes: EdgeChange[]) => { + onChange({ nodes: graph.nodes, edges: applyEdgeChanges(changes, graph.edges) }) + }, + [graph.nodes, graph.edges, onChange] + ) + const onConnect = useCallback( + (connection: Connection) => { + const id = `e_${crypto.randomUUID()}` + onChange({ nodes: graph.nodes, edges: [...graph.edges, { id, ...connection } as any] }) + }, + [graph.nodes, graph.edges, onChange] + ) + + const updateNodeConfig = useCallback( + (id: string, config: any) => { + const nodes = graph.nodes.map((n) => (n.id === id ? { ...n, data: { ...(n.data || {}), config } } : n)) + onChange({ nodes, edges: graph.edges }) + }, + [graph.nodes, graph.edges, onChange] + ) + + return ( +
+
+ + + + + + + +
+ {showLeft && ( + +
+
+
Library
+ +
+ {/* Library list placeholder - controlled editor expects external library drag/drop wiring */} +
Drag blocks from external library…
+
+ + )} + {showRight && ( + +
+
+
Inspector
+ +
+
Select a node to edit…
+
+ + )} + {showBottom && ( + +
+
+
Output
+ +
+
Outputs appear here…
+
+ + )} +
+ + +
+
+ ) +} diff --git a/src/tools/new/components/app/AppBar.tsx b/src/tools/new/components/app/AppBar.tsx new file mode 100644 index 0000000..d8937a0 --- /dev/null +++ b/src/tools/new/components/app/AppBar.tsx @@ -0,0 +1,32 @@ +import React from 'react' +import { useGraphActions } from '../../logic/useGraphActions' + +export const AppBar: React.FC = () => { + const { runAll, openExport } = useGraphActions() + return ( +
+
+ Pipeline +
+ + + +
+
+
+ +
+ +
+
+
+ ) +} diff --git a/src/tools/new/components/app/AppShell.tsx b/src/tools/new/components/app/AppShell.tsx new file mode 100644 index 0000000..0362c46 --- /dev/null +++ b/src/tools/new/components/app/AppShell.tsx @@ -0,0 +1,23 @@ +import React from 'react' +import { GraphWorkbench } from '../graph/GraphWorkbench' +import { AppBar } from './AppBar' +import { BottomPanel } from './BottomPanel' +import { LeftSidebar } from './LeftSidebar' +import { RightPane } from './RightPane' + +export const AppShell: React.FC = () => { + return ( +
+ +
+ +
+ +
+ +
+ + {/* Toasts and Command Palette hooks can be added later */} +
+ ) +} diff --git a/src/tools/new/components/app/BottomPanel.tsx b/src/tools/new/components/app/BottomPanel.tsx new file mode 100644 index 0000000..a0e467a --- /dev/null +++ b/src/tools/new/components/app/BottomPanel.tsx @@ -0,0 +1,31 @@ +import React from 'react' +import { useOutput } from '../../state/outputState' + +export const BottomPanel: React.FC = () => { + const { datatype, data, exportAs } = useOutput() + return ( +
+
+ Output +
+ + + +
+
+ {datatype === 'table' && 
{JSON.stringify(data, null, 2)}
} + {datatype === 'json' && 
{JSON.stringify(data, null, 2)}
} + {datatype === 'geojson' && ( + 
{JSON.stringify(data, null, 2)}
+ )} + {datatype === 'chart' && 
{JSON.stringify(data, null, 2)}
} +
+
+ ) +} diff --git a/src/tools/new/components/app/LeftSidebar.tsx b/src/tools/new/components/app/LeftSidebar.tsx new file mode 100644 index 0000000..11cca97 --- /dev/null +++ b/src/tools/new/components/app/LeftSidebar.tsx @@ -0,0 +1,37 @@ +import React from 'react' +import { useLibrary } from '../../state/libraryState' + +export const LeftSidebar: React.FC = () => { + const { groups, filter, setFilter, startDrag } = useLibrary() + + return ( + + ) +} diff --git a/src/tools/new/components/app/RightPane.tsx b/src/tools/new/components/app/RightPane.tsx new file mode 100644 index 0000000..fae0c3e --- /dev/null +++ b/src/tools/new/components/app/RightPane.tsx @@ -0,0 +1,48 @@ +import React from 'react' +import { useGraphStore } from '../../state/graphState' +import { useSelection } from '../../state/selectionState' + +export const RightPane: React.FC = () => { + const { selectedNodeId, nodeConfig, setNodeConfig, logs, clearLogs } = useSelection() + const { updateNodeConfig } = useGraphStore() + return ( +