diff --git a/.gitignore b/.gitignore index f149c73..efd1034 100755 --- a/.gitignore +++ b/.gitignore @@ -113,4 +113,6 @@ index.ts.bak public/projects/* -public/data/* \ No newline at end of file +public/data/* + +storage/ \ No newline at end of file diff --git a/fileserver.ts b/fileserver.ts new file mode 100644 index 0000000..99617b7 --- /dev/null +++ b/fileserver.ts @@ -0,0 +1,184 @@ +import express, { Request } from 'express' +import fs from 'fs' +import https from 'https' +import path from 'path' + +type QueryParams = { + source: string + predicate: string +} + +type FileParams = QueryParams & { + target: string +} + +// Basic config +const PORT = 8000 +// Resolve repo-level dev certs; this file lives in packages/projects/projects/hexafield/conjure/ +const CERT_PATH = path.resolve(__dirname, '../../../../../certs/cert.pem') +const KEY_PATH = path.resolve(__dirname, '../../../../../certs/key.pem') + +// Local storage directory for uploaded files +const STORAGE_DIR = path.resolve(__dirname, 'storage') +fs.mkdirSync(STORAGE_DIR, { recursive: true }) + +// Minimal CORS for local dev +const corsMiddleware: express.RequestHandler = (req, res, next) => { + console.log(`[fileserver] ${req.method} ${req.url}`) + res.header('Access-Control-Allow-Origin', req.headers.origin || '*') + res.header('Access-Control-Allow-Credentials', 'true') + res.header('Access-Control-Allow-Methods', 'GET,POST,OPTIONS') + res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization') + if (req.method === 'OPTIONS') return res.sendStatus(204) + next() +} + +const app = express() +app.use(corsMiddleware) +app.use(express.json({ limit: '10mb' })) // JSON only for metadata routes + +const keyFor = ({ source, predicate }: QueryParams) => `${encodeURIComponent(predicate)}__${encodeURIComponent(source)}` + +const pathFor = (q: QueryParams) => path.join(STORAGE_DIR, `${keyFor(q)}.json`) + +const readFile = (q: QueryParams): FileParams | undefined => { + const path = pathFor(q) + if (!fs.existsSync(path)) return undefined + try { + return JSON.parse(fs.readFileSync(path, 'utf-8')) as FileParams + } catch { + return undefined + } +} + +const writeFile = (q: QueryParams, data: string) => { + fs.writeFileSync(pathFor(q), data) +} + +app.get('/health', (_req, res) => res.status(200).send('ok')) + +app.post('/create', (req: Request, res) => { + try { + const { source, predicate, target } = req.body as unknown as FileParams + if (!source || !predicate || !target) { + return res.status(400).json({ error: 'Missing required fields' }) + } + + const q: QueryParams = { source, predicate } + const finalPath = pathFor(q) + + if (finalPath && fs.existsSync(finalPath)) fs.unlinkSync(finalPath) + + writeFile(q, target) + + return res.status(201).json({ ok: true }) + } catch (e) { + console.error('CREATE error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// Get (download) +app.post('/get', (req, res) => { + try { + const { source, predicate } = req.body as QueryParams + if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' }) + + const path = pathFor({ source, predicate }) + if (!path || !fs.existsSync(path)) return res.status(404).json({ error: 'Not found' }) + + const file = readFile({ source, predicate }) + if (!file) return res.status(404).json({ error: 'Not found' }) + + return res.status(200).json(file) + } catch (e) { + console.error('GET error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// Find (return results for matching predicate) +app.post('/find', (req, res) => { + try { + const { predicate } = req.body as { predicate?: string } + if (!predicate) return res.status(400).json({ error: 'Missing required fields' }) + + // Scan storage dir for all .json meta files + const results: Array = [] + const files = fs.readdirSync(STORAGE_DIR) + for (const file of files) { + if (!file.endsWith('.json')) continue + const encodedPredicate = encodeURIComponent(predicate) + if (file.startsWith(encodedPredicate)) { + results.push(decodeURIComponent(file.slice(encodedPredicate.length + 2, -5))) + } + } + return res.status(200).json({ ok: true, results }) + } catch (e) { + console.error('FIND error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +app.post('/has', (req, res) => { + try { + const { source, predicate } = req.body as QueryParams + if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' }) + + const file = fs.existsSync(pathFor({ source, predicate })) + if (!file) return res.status(200).json({ ok: false }) + + return res.status(200).json({ ok: true }) + } catch (e) { + console.error('HAS error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// Replace +app.post('/replace', (req: Request, res) => { + try { + const { source, predicate, target } = req.body as unknown as FileParams + if (!source || !predicate || !target) { + return res.status(400).json({ error: 'Missing required fields' }) + } + const q: QueryParams = { source, predicate } + const path = pathFor(q) + if (!path) { + return res.status(404).json({ error: 'Not found' }) + } + // Remove old file + if (path && fs.existsSync(path)) fs.unlinkSync(path) + + writeFile(q, target) + + return res.status(200).json({ ok: true }) + } catch (e) { + console.error('REPLACE error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// Delete +app.post('/delete', (req, res) => { + try { + const { source, predicate } = req.body as QueryParams + if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' }) + const q: QueryParams = { source, predicate } + + const path = pathFor(q) + if (fs.existsSync(path)) fs.unlinkSync(path) + + return res.status(200).json({ ok: true }) + } catch (e) { + console.error('DELETE error', e) + return res.status(500).json({ error: 'Internal error' }) + } +}) + +// HTTPS server +const key = fs.readFileSync(KEY_PATH) +const cert = fs.readFileSync(CERT_PATH) +https.createServer({ key, cert }, app).listen(PORT, () => { + console.log(`[fileserver] HTTPS listening on https://localhost:${PORT}`) +}) diff --git a/package.json b/package.json index f195668..758a209 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "build:worker": "esbuild src/tools/graph/forcegraph/ForceGraphWorker.ts --bundle --format=esm --outfile=src/tools/graph/forcegraph/ForceGraphWorker.bundle.js --loader:.wasm=binary", "build:dag-worker": "esbuild src/tools/graph/dag/DagWorker.ts --bundle --format=esm --outfile=src/tools/graph/dag/DagWorker.bundle.js --loader:.wasm=binary", "start": "tsx server.js", + "fileserver": "tsx fileserver.ts", "format": "prettier --write \"**/*.{ts,tsx}\"", "format-scss": "stylelint \"**/*.scss\" --fix", "format-staged": "lint-staged", @@ -24,13 +25,18 @@ "@coasys/ad4m-connect": "^0.10.1-release-candidate-4", "@hexafield/jsonpath-object-transform": "^2.0.0", "@mlc-ai/web-llm": "^0.2.79", + "canonicalize": "^2.1.0", "d3": "7.9.0", "d3-force-3d": "^3.0.5", "dat.gui": "^0.7.9", + "express": "^4.19.2", + "json-schema-library": "^10.2.1", "json-schema-to-ts": "^3.1.1", "jsonpath-plus": "10.3.0", "maxrects-packer": "2.7.3", "monaco-editor": "^0.52.2", + "react-rnd": "^10.5.2", + "reactflow": "^11.10.0", "ts-morph": "^26.0.0" }, "license": "ISC", diff --git a/src/ad4m/useADAM.tsx b/src/ad4m/useADAM.tsx index c123ac7..ec2396c 100644 --- a/src/ad4m/useADAM.tsx +++ b/src/ad4m/useADAM.tsx @@ -1,7 +1,36 @@ -import { Ad4mClient, Agent } from '@coasys/ad4m' +import { Ad4mClient, Agent, Link, LinkMutations, Literal } from '@coasys/ad4m' import Ad4mConnectUI, { Ad4mConnectElement, getAd4mClient } from '@coasys/ad4m-connect' -import { defineState, getMutableState, useHookstate, useMutableState } from '@ir-engine/hyperflux' +import { defineState, getMutableState, getState, useHookstate, useMutableState } from '@ir-engine/hyperflux' import { useEffect } from 'react' +import { CRUD_API, P2P_API } from '../api/CRUD' + +export const blobToDataURL = (blob: Blob): Promise => { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => { + const result = reader.result as string + const base64 = result.split(',')[1] + resolve(base64 as string) + } + reader.onerror = () => reject(reader.error) + reader.onabort = () => reject(new Error('Read aborted')) + reader.readAsDataURL(blob) + }) +} + +export const dataURLToBlob = (dataURL: string): Promise => { + return new Promise((resolve, reject) => { + const arr = dataURL.split(',') + const mime = arr[0].match(/:(.*?);/)?.[1] + const bstr = atob(arr[1]) + let n = bstr.length + const u8arr = new Uint8Array(n) + while (n--) { + u8arr[n] = bstr.charCodeAt(n) + } + resolve(new Blob([u8arr], { type: mime })) + }) +} export const AdamClientState = defineState({ name: 'hexafield.adam-template.AdamClientState', @@ -69,7 +98,123 @@ export const AgentState = defineState({ adam.agent.me().then((response) => { getMutableState(AgentState).set(response) + P2P_API.client = AgentBlobAPI + getMutableState(P2P_API).ready.set(true) + console.log('ADAM storage method ready') }) }, [adam]) } }) + +export const AgentBlobAPI: CRUD_API = { + create: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const newLink = new Link({ + source: args.source, + predicate: args.predicate, + target: Literal.from(args.target).toUrl() + }) + + await client.agent.mutatePublicPerspective({ + additions: [newLink], + removals: [] + } as LinkMutations) + }, + + has: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') + + const myLinks = myPerspectives.perspective!.links + + const link = myLinks.find((link) => link.data.source === args.source && link.data.predicate === args.predicate) + return !!link + }, + + get: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') + + const myLinks = myPerspectives.perspective!.links + + const link = myLinks.find((link) => link.data.source === args.source && link.data.predicate === args.predicate) + if (!link) return undefined // not found + + return Literal.fromUrl(link.data.target).get() + }, + + find: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') + + const myLinks = myPerspectives.perspective!.links + + const foundLinks = myLinks.filter((link) => link.data.predicate === args.predicate) + return foundLinks.map((link) => link.data.source) + }, + + replace: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') + + const myLinks = myPerspectives.perspective!.links + + const currentLink = myLinks.find( + (link) => link.data.source === args.source && link.data.predicate === args.predicate + ) + if (!currentLink) throw new Error('Current link not found') + + const removals = [currentLink] + + const newLink = new Link({ + source: args.source, + predicate: args.predicate, + target: Literal.from(args.target).toUrl() + }) + + await client.agent.mutatePublicPerspective({ + additions: [newLink], + removals: removals + }) + }, + + delete: async (args) => { + const client = getState(AdamClientState) + if (!client) throw new Error('AD4M client not initialized') + + const myPerspectives = await client.agent.me() + if (!myPerspectives) throw new Error('Agent not initialized') + + const myLinks = myPerspectives.perspective!.links + + const currentLink = myLinks.find( + (link) => link.data.source === args.source && link.data.predicate === args.predicate + ) + if (!currentLink) throw new Error('Current link not found') + + await client.agent.mutatePublicPerspective({ + additions: [], + removals: [currentLink] + }) + } +} + +globalThis.AdamClientState = AdamClientState +globalThis.AgentState = AgentState +globalThis.AgentBlobAPI = AgentBlobAPI + +// wipe data: `AgentBlobAPI.find({ predicate: 'conjure://schema' }).then(sources => sources.forEach(source => AgentBlobAPI.delete({ source, predicate: 'conjure://schema' })))` diff --git a/src/api/CRUD.ts b/src/api/CRUD.ts new file mode 100644 index 0000000..24127da --- /dev/null +++ b/src/api/CRUD.ts @@ -0,0 +1,28 @@ +import { defineState, syncStateWithLocalStorage } from '@ir-engine/hyperflux' + +export type QueryParams = { + source: string + predicate: string +} + +export type RDFParams = QueryParams & { + target: TargetType +} + +export type TargetType = string | object | boolean | number + +export type CRUD_API = { + create: (args: RDFParams) => Promise + get: (args: QueryParams) => Promise + has: (args: QueryParams) => Promise + find: (args: { predicate: string }) => Promise // Find files by predicate, returns list of sources + replace: (args: RDFParams) => Promise + delete: (args: QueryParams) => Promise +} + +export const P2P_API = defineState({ + name: 'hexafield.conjure.P2P_API', + initial: { ready: false, selected: 'local' as 'local' | 'server' | 'adam' }, + client: null! as CRUD_API, + extension: syncStateWithLocalStorage(['selected']) +}) diff --git a/src/api/local.ts b/src/api/local.ts new file mode 100644 index 0000000..d3b9ac2 --- /dev/null +++ b/src/api/local.ts @@ -0,0 +1,78 @@ +import type { CRUD_API, QueryParams } from './CRUD' + +const conjureCacheKey = 'CONJURE_STORAGE_CACHE' + +const localCache = { entries: {} as Record, predicateIndex: {} as Record } +globalThis.localCache = localCache + +const saveCache = () => { + localStorage.setItem(conjureCacheKey, JSON.stringify(localCache.entries)) +} + +const loadCache = () => { + const cached = localStorage.getItem(conjureCacheKey) + if (cached) { + const parsed = JSON.parse(cached) + for (const key in parsed) { + localCache.entries[key] = parsed[key] + } + } + localCache.predicateIndex = Object.keys(localCache.entries).reduce( + (acc, key) => { + const [predicate] = key.split('__') + if (!acc[predicate]) { + acc[predicate] = [] + } + acc[predicate].push(key) + return acc + }, + {} as Record + ) +} + +loadCache() + +const keyFor = ({ source, predicate }: QueryParams) => `${encodeURIComponent(predicate)}__${encodeURIComponent(source)}` + +export const LocalBlobAPI: CRUD_API = { + create: async (args) => { + const key = keyFor(args) + localCache.entries[key] = JSON.stringify(args.target) + saveCache() + }, + + get: async (args) => { + const key = keyFor(args) + const value = localCache.entries[key] + return value ? JSON.parse(value) : undefined + }, + + has: async (args) => { + const key = keyFor(args) + return localCache.entries[key] !== undefined + }, + + find: async (args) => { + const encodedPredicate = encodeURIComponent(args.predicate) + const results = [] as string[] + for (const key of localCache.predicateIndex[encodedPredicate] || []) { + const value = localCache.entries[key] + if (typeof value === 'string') { + results.push(decodeURIComponent(key.slice(encodedPredicate.length + 2))) + } + } + return results + }, + + replace: async (args) => { + const key = keyFor(args) + localCache.entries[key] = JSON.stringify(args.target) + saveCache() + }, + + delete: async (args) => { + const key = keyFor(args) + delete localCache.entries[key] + saveCache() + } +} diff --git a/src/api/server.ts b/src/api/server.ts new file mode 100644 index 0000000..10cc987 --- /dev/null +++ b/src/api/server.ts @@ -0,0 +1,71 @@ +import type { CRUD_API } from './CRUD' + +const url = 'https://localhost:8000' + +export const ServerBlobAPI: CRUD_API = { + create: async (args) => { + const response = await fetch(`${url}/create`, { + method: 'POST', + body: JSON.stringify({ + source: args.source, + predicate: args.predicate, + target: JSON.stringify(args.target) + }), + headers: { 'Content-Type': 'application/json' } + }) + if (!response.ok) throw new Error('Failed to create blob') + }, + + get: async (args) => { + const response = await fetch(`${url}/get`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (response.status === 404) return undefined + if (!response.ok) throw new Error('Failed to get blob') + const json = await response.json() + return json.target + }, + + has: async (args) => { + const response = await fetch(`${url}/has`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + return (await response.json()).ok + }, + + find: async (args) => { + const response = await fetch(`${url}/find`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (!response.ok) throw new Error('Failed to find blobs') + return (await response.json()).results + }, + + replace: async (args) => { + const response = await fetch(`${url}/replace`, { + method: 'POST', + body: JSON.stringify({ + source: args.source, + predicate: args.predicate, + target: JSON.stringify(args.target) + }), + headers: { 'Content-Type': 'application/json' } + }) + if (!response.ok) throw new Error('Failed to replace blob') + }, + + delete: async (args) => { + const response = await fetch(`${url}/delete`, { + method: 'POST', + body: JSON.stringify(args), + headers: { 'Content-Type': 'application/json' } + }) + if (response.status !== 404 && !response.ok) throw new Error('Failed to delete blob') + } +} diff --git a/src/tools/GraphPage.tsx b/src/tools/GraphPage.tsx index abe7e1d..72b44b8 100644 --- a/src/tools/GraphPage.tsx +++ b/src/tools/GraphPage.tsx @@ -2,13 +2,17 @@ import '@ir-engine/client/src/engine' import { Resizable } from 're-resizable' import React, { useEffect } from 'react' +import { useDrop } from 'react-dnd' +import { NativeTypes } from 'react-dnd-html5-backend' import { HiChevronLeft, HiChevronRight } from 'react-icons/hi' import { Vector3 } from 'three' import Debug from '@ir-engine/client-core/src/components/Debug' import { useDraggable } from '@ir-engine/client-core/src/hooks/useDraggable' import { createEntity, EntityTreeComponent, removeEntity, setComponent } from '@ir-engine/ecs' -import { getMutableState, useHookstate, useMutableState, useReactiveRef } from '@ir-engine/hyperflux' +import { DndWrapper } from '@ir-engine/editor/src/components/dnd/DndWrapper' +import { DnDFileType } from '@ir-engine/editor/src/constants/AssetTypes' +import { getMutableState, getState, useHookstate, useMutableState, useReactiveRef } from '@ir-engine/hyperflux' import { AmbientLightComponent, ReferenceSpaceState, TransformComponent } from '@ir-engine/spatial' import { CameraOrbitComponent } from '@ir-engine/spatial/src/camera/components/CameraOrbitComponent' import { NameComponent } from '@ir-engine/spatial/src/common/NameComponent' @@ -18,14 +22,21 @@ import { useEngineCanvas } from '@ir-engine/spatial/src/renderer/functions/useEn import { RendererState } from '@ir-engine/spatial/src/renderer/RendererState' import { Button } from '@ir-engine/ui' +import { P2P_API } from '../api/CRUD' import GithubLink from './components/GithubLink' +import Tabs from './components/Tabs' +import { contentHashJSONSchema } from './json-schema/contentHash' +import { JSONSchemaType } from './json-schema/JSONSchema' +import { callLLM, CODING_MODELS } from './llm/useLLM' +import { SchemaRegistry } from './registries/SchemaRegistry' +import { TargetRegistry } from './registries/TargetRegistry' +import { Stringify, ToolRegistry } from './registries/ToolRegistry' import { PipelineView } from './views/PipelineView' import SchemaView from './views/SchemaView' import ToolView from './views/ToolView' -import Tabs from './components/Tabs' - import './graph/forcegraph/ForceGraph' +import { createJSONTransformFunctionPrompt } from './json-schema/createJSONTransformFunctionPrompt' const tabs = [ { label: 'Pipelines', value: 'pipeline' }, @@ -49,7 +60,35 @@ function ToolMenus(): JSX.Element { } function ToolUI() { + const storageMethod = useHookstate(getMutableState(P2P_API).selected) const showMappingUI = useHookstate(true) + + useEffect(() => { + if (storageMethod.value === 'adam') { + import('../ad4m/useADAM').then((module) => { + getMutableState(P2P_API).ready.set(false) + P2P_API.client = module.AgentBlobAPI + // get agent state to initialize agent + if (getState(module.AgentState)) { + getMutableState(P2P_API).ready.set(true) + console.log('ADAM storage method ready') + } + }) + } else if (storageMethod.value === 'server') { + import('../api/server').then((module) => { + P2P_API.client = module.ServerBlobAPI + getMutableState(P2P_API).ready.set(true) + console.log('Server storage method ready') + }) + } else if (storageMethod.value === 'local') { + import('../api/local').then((module) => { + P2P_API.client = module.LocalBlobAPI + getMutableState(P2P_API).ready.set(true) + console.log('Local storage method ready') + }) + } + }, [storageMethod.value]) + const size = useHookstate<{ width: number; height: number }>(() => { const w = Number.parseInt(localStorage.getItem('toolUIWidth') || '') const h = Number.parseInt(localStorage.getItem('toolUIHeight') || '') @@ -119,13 +158,7 @@ function ToolUI() { )}

Tool Menu

- {showMappingUI.value && ( -
- -
- )} + @@ -168,11 +201,160 @@ export default function GraphPage() { }, [originEntity, viewerEntity]) return ( - <> -
+
+ + +
+ + - +
+ ) +} + +const GraphDND = ({ children }: { children: React.ReactNode }) => { + const [{ isDragging }, dropRef] = useDrop({ + accept: ['json', NativeTypes.FILE], + collect: (monitor) => ({ + isDragging: monitor.getItem() !== null && monitor.canDrop() && monitor.isOver() + }), + drop: async (item: DnDFileType, monitor) => { + if (!('files' in item)) return + const files = (monitor.getItem() as DataTransfer).files + if (!files.length) return + const [outputHash, outputSchema] = Object.entries(getState(TargetRegistry))[0] // for now, just hardcode the only output target we have + + const transformedDataBySource = Object.fromEntries( + await Promise.all( + [...files].map(async (file) => { + const json = JSON.parse(await file.text()) + const inputSchema = SchemaRegistry.findOrGenerateMatchingSchema( + json, + file.name, + `Generated schema from ${file.name}` + ) + + const toolExists = Object.entries(getState(ToolRegistry).tools).find(([key, value]) => { + return value.inputHash === inputSchema.hash && value.outputHash === outputSchema.hash + }) + let toolHash = toolExists?.[0] + if (!toolExists) { + console.log('Generating JSON transformer for', file.name) + const result = await callLLM( + { + prompt: createJSONTransformFunctionPrompt({ + inputSchema: inputSchema.schema as JSONSchemaType, + outputSchema: outputSchema.value, + additionalInstructions: + 'If a schema specifies that it allows additional properties, include them in the output, attempting to map them to known properties when specified.' + }), + output: 'javascript' + }, + { modelId: CODING_MODELS[0].id } + ) + const cleanResponse = result!.rawResponse + .replace('```javascript', '') + .replace('```', '') + .replace('\\n', '\n') as Stringify<(input: unknown) => Promise> + toolHash = await ToolRegistry.create({ + label: `${inputSchema.label} to ${outputSchema.label}`, + description: `Converts data from ${inputSchema.label} to ${outputSchema.label}`, + input: inputSchema.schema as JSONSchemaType, + output: outputSchema.value as JSONSchemaType, + transformation: cleanResponse + }) + } + console.log(getState(ToolRegistry).tools[toolHash!]) + console.log(json) + const transformedData = await ToolRegistry.run< + JSONSchemaType, + JSONSchemaType + >(toolHash!, json) + console.log(transformedData) + return [file.name, transformedData] as const + }) + ) + ) + if (files.length > 1) { + const outputRecordSchema = { + type: 'object' as const, + additionalProperties: outputSchema.value as JSONSchemaType + } + + const outputRecordSchemaHash = contentHashJSONSchema( + outputRecordSchema as any /**@todo unify json schema types */ + ) + if (!getState(SchemaRegistry).schemas[outputRecordSchemaHash]) { + SchemaRegistry.register( + outputRecordSchema, + `Multi-source ${outputSchema.label}`, + `Multiple source keyed record for ${outputSchema.label}` + ) + } + + const toolExists = Object.entries(getState(ToolRegistry).tools).find(([key, value]) => { + return value.inputHash === outputRecordSchemaHash && value.outputHash === outputSchema.hash + }) + let sourceCombineToolHash = toolExists?.[0] + if (!toolExists) { + const result = await callLLM( + { + prompt: createJSONTransformFunctionPrompt({ + inputSchema: outputRecordSchema, + outputSchema: outputSchema.value, + additionalInstructions: `This transformer must merge multiple sources of the same data shape into one output object, as defined by the output schema, preserving extraneous properties and metadata. +If there is a field or additional properties allowed on the output schema that specifies a source or sources, include it in the output. +If a schema specifies that it allows additional properties, include them in the transformation, attempting to map them to known properties when specified, or simply including them as is. +If there are entries across multiple sources that refer to the same thing (by semantic meaning, such as a unique name, rather than just an ID), these data points should be merged, with relationships preserved.` + }), + output: 'javascript' + }, + { modelId: CODING_MODELS[0].id } + ) + + const cleanResponse = result!.rawResponse + .replace('```javascript', '') + .replace('```', '') + .replace('\\n', '\n') as Stringify<(input: unknown) => Promise> + sourceCombineToolHash = await ToolRegistry.create({ + label: `Deduplicates Multi-Source ${outputSchema.label}`, + description: `Combines multiple sources into a single output for ${outputSchema.label}, deduplicating common entries and grouping by originating source.`, + input: outputRecordSchema as JSONSchemaType as JSONSchemaType, + output: outputSchema.value as JSONSchemaType, + transformation: cleanResponse + }) + } + + const transformedData = await ToolRegistry.run(sourceCombineToolHash!, transformedDataBySource) + console.log({ transformedDataBySource, transformedData }) + await outputSchema.deserialize(transformedData) + } else { + await outputSchema.deserialize(transformedDataBySource[files[0].name]) + } + } + }) + + return ( +
+ {children} +
+ ) +} + +const StorageDropdown = ({ + storageMethod, + onChange +}: { + storageMethod: { value: 'local' | 'server' | 'adam' } + onChange: (value: 'local' | 'server' | 'adam') => void +}) => { + return ( + ) } diff --git a/src/tools/components/PipelineCard.tsx b/src/tools/components/PipelineCard.tsx new file mode 100644 index 0000000..08a5b55 --- /dev/null +++ b/src/tools/components/PipelineCard.tsx @@ -0,0 +1,50 @@ +import React, { useMemo, useState } from 'react' +import { graphToPipeline, pipelineToGraph } from '../pipeline/graphConvert' +import { PipelineSpec } from '../pipeline/model' +import { PipelineEditor, PipelineGraph } from '../pipeline/PipelineEditor' +import type { Pipeline } from '../registries/PipelineRegistry' + +type ToolLite = { hash: string; label: string } + +type Props = { + pipeline: Pipeline + tools: ToolLite[] + onRun: () => void + onSaveGraph: (spec: PipelineSpec, pipeline: Pipeline) => void +} + +export const PipelineCard: React.FC = ({ pipeline, tools, onRun, onSaveGraph }) => { + const [open, setOpen] = useState(false) + const initialGraph = useMemo(() => { + return pipelineToGraph(pipeline.graph) + }, [pipeline.hash]) + const [working, setWorking] = useState(initialGraph) + + const save = (g: PipelineGraph) => { + const spec = graphToPipeline(g.nodes, g.edges) + onSaveGraph(spec, pipeline) + } + + return ( +
+
+
+ {pipeline.label} + {pipeline.hash.slice(0, 8)} +
+
+ +
+
+ {open && ( +
+ +
+ )} +
+ ) +} + +export default PipelineCard diff --git a/src/tools/components/ToolCard.tsx b/src/tools/components/ToolCard.tsx index c54e2b8..c675617 100644 --- a/src/tools/components/ToolCard.tsx +++ b/src/tools/components/ToolCard.tsx @@ -5,14 +5,24 @@ import { JsonDisplay } from './JsonDisplay' interface ToolCardProps { tool: Tool onUse?: (tool: Tool) => void + onForget?: (hash: string) => void } -export const ToolCard = ({ tool, onUse }: ToolCardProps) => { +export const ToolCard = ({ tool, onUse, onForget }: ToolCardProps) => { return (
  • {tool.label} {tool.hash.slice(0, 8)}... + {onForget && ( + + )}
    {tool.description}
    { ) } -const forcegraphSchema: JSONSchemaType = { +const forcegraphSchema = { type: 'object', + description: 'Data format for a rich force graph, with support for parameterized layout and styling options.', required: ['nodes', 'edges'], properties: { nodes: { type: 'array', items: { type: 'object', + description: + 'A node in the force graph. Can accept additional arbitrary data per node, such as images, descriptions or URLs.', required: ['id', 'label'], properties: { - id: { type: 'string' }, - label: { type: 'string' }, - group: { type: 'string', nullable: true, default: '' }, - image: { type: 'string', nullable: true, default: '' } - } + id: { type: 'number', description: 'The unique identifier for the node.' }, + label: { type: 'string', description: 'The label of the node.' }, + group: { type: 'string', description: 'The group to which the node belongs.' } + }, + additionalProperties: true } }, edges: { type: 'array', items: { type: 'object', + description: 'An edge between two nodes in the force graph. Can accept additional arbitrary data per edge.', required: ['source', 'target'], properties: { - source: { type: 'string' }, - target: { type: 'string' }, - weight: { type: 'number', nullable: true, default: 1 } + source: { type: 'number', description: 'The ID of the source node.' }, + target: { type: 'number', description: 'The ID of the target node.' }, + type: { type: 'string', description: 'The type of the edge.' }, + weight: { type: 'number', nullable: true, default: 1, description: 'The weight of the edge.' }, + additionalProperties: true } } } @@ -536,93 +539,41 @@ const forcegraphSchema: JSONSchemaType = { // default: 1 // } } -} +} as const satisfies JSONSchema -type SerializedForceGraphShape = { - nodes: Array<{ - id: string | number - label: string - group?: string - image?: string - }> - edges: Array<{ - source: string | number - target: string | number - weight?: number - }> -} +type SerializedForceGraphShape = FromSchema export const ForceGraphSchema: TargetSchemaDefinition = { label: 'Force Graph', - value: forcegraphSchema, + value: forcegraphSchema as any, // @todo unify json schema typing deserialize: (data) => { - const finalData: SerializedForceGraphShape = { nodes: [], edges: [] } - const seenLabels = new Map() - const replacedNodes = {} as Record> + console.log(data) let maxWeight = 0 - for (const sourceID in data) { - const source = data[sourceID] - if (typeof source !== 'object') continue - //sum the various data sources together, distinguishing different sources by the 'category' field on nodes - /** @todo this should be a configurable field in the mapping UI - need support for a 'literal' either at the source or overall level */ - if (Array.isArray(source.nodes)) { - for (let i = 0; i < source.nodes.length; i++) { - const node = source.nodes[i] - const seenNode = seenLabels.get(node.label) - if (seenNode) { - if (!replacedNodes[sourceID]) { - replacedNodes[sourceID] = new Map() - } - replacedNodes[sourceID].set(node.id, seenNode.id) - } else { - seenLabels.set(node.label, { source: sourceID, id: node.id }) - finalData.nodes.push({ - ...node, - group: sourceID - }) - } - } - } - if (Array.isArray(source.edges)) { - for (const edge of source.edges) { - finalData.edges.push({ - source: replacedNodes[sourceID]?.get(edge.source) ?? edge.source, - target: replacedNodes[sourceID]?.get(edge.target) ?? edge.target, - weight: edge.weight - }) - } - } - } + // ensure all edges have a weight - for (const edge of finalData.edges) { + for (const edge of data.edges) { edge.weight = edge.weight || 1 maxWeight = Math.max(maxWeight, edge.weight) } - const minConnections = 1 - // quick hack, remove all nodes that only have one edge - const nodeCounts = new Map() - for (const edge of finalData.edges) { - nodeCounts.set(edge.source, (nodeCounts.get(edge.source) || 0) + 1) - nodeCounts.set(edge.target, (nodeCounts.get(edge.target) || 0) + 1) + for (const edge of data.edges) { // scale all weights between 0 and 1 edge.weight = edge.weight! / maxWeight } - finalData.nodes = finalData.nodes.filter( - (node) => nodeCounts.get(node.id) && nodeCounts.get(node.id)! >= minConnections - ) // and now remove all edges that don't have both nodes - finalData.edges = finalData.edges.filter( - (edge) => - finalData.nodes.find((node) => node.id === edge.source) && - finalData.nodes.find((node) => node.id === edge.target) - ) + data.edges = data.edges.filter((edge) => { + if (data.nodes.find((node) => node.id === edge.source) && data.nodes.find((node) => node.id === edge.target)) { + return true + } + console.warn('removing edge', edge) + return false + }) - if (!finalData.nodes.length) return null! + if (!data.nodes.length) return null! - getMutableState(d3State).nodes.set(finalData.nodes as NodeData['nodes']) - getMutableState(d3State).links.set(finalData.edges as NodeData['edges']) + getMutableState(d3State).nodes.set(data.nodes as NodeData['nodes']) + getMutableState(d3State).links.set(data.edges as NodeData['edges']) } } @@ -638,73 +589,73 @@ export const ForceGraphSchema: TargetSchemaDefinition // ...dataset.nodes // ] -const atlasImages = async (nodesWithDefaultImage: NodeData['nodes']) => { - const images = await Promise.all( - nodesWithDefaultImage.map((node) => { - return new Promise((resolve) => { - if (!node.imageSrc) return resolve(null) - const image = new Image() - image.onload = () => { - resolve(image) - } - image.crossOrigin = 'Anonymous' - image.onerror = (e) => { - resolve(null!) - console.log('failed to load image', node.imageSrc, e) - } - image.src = `https://cors-anywhere.herokuapp.com/${node.imageSrc!}` - }) - // take loaded images and convert them into an atlas map. assume the images are the square and the same size - }) - ) - - const imagesLoaded = images.filter((i) => !!i) - const atlasSize = Math.ceil(Math.sqrt(imagesLoaded.length)) - const atlas = document.createElement('canvas') - const resolution = 128 - atlas.width = atlasSize * resolution - atlas.height = atlasSize * resolution - const ctx = atlas.getContext('2d')! - const indicies = new Set() - imagesLoaded.forEach((image, i) => { - // flip image - const x = i % atlasSize - const y = Math.floor(i / atlasSize) - ctx.drawImage(image!, x * resolution, y * resolution, resolution, resolution) - indicies.add(i) - }) - //convert the atlas to imagedata and then a blob - const imageData = ctx.getImageData(0, 0, atlas.width, atlas.height) - - const texture = new DataTexture(imageData.data, atlas.width, atlas.height) - texture.colorSpace = SRGBColorSpace - texture.needsUpdate = true - // texture.flipY = true - const material = new MeshBasicMaterial({ map: texture, side: BackSide }) - const circleGeom = new CircleGeometry(graphScale, 16).scale(1, -1, 1) // unflip y - - const uvOffset = new Float32Array(images.length * 2) - // set the uv offset for each image, accounting for images that are not loaded - for (let i = 0; i < images.length; i++) { - const isLoaded = indicies.has(i) - if (!isLoaded) continue - // start from top left - const x = i % atlasSize - const y = Math.floor(i / atlasSize) - uvOffset[i * 2] = x / atlasSize - uvOffset[i * 2 + 1] = y / atlasSize - } - circleGeom.setAttribute('uvOffset', new InstancedBufferAttribute(uvOffset, 2)) - - material.onBeforeCompile = function (shader) { - shader.vertexShader = shader.vertexShader.replace('void main() {', 'attribute vec2 uvOffset;\n' + 'void main() {') - - shader.vertexShader = shader.vertexShader.replace( - '#include ', - '#include \n' + `vMapUv = (uv * ${1 / atlasSize}) + uvOffset;` - ) - } - - const mesh = new InstancedMesh(circleGeom, material, images.length) - mesh.frustumCulled = false -} +// const atlasImages = async (nodesWithDefaultImage: NodeData['nodes']) => { +// const images = await Promise.all( +// nodesWithDefaultImage.map((node) => { +// return new Promise((resolve) => { +// if (!node.imageSrc) return resolve(null) +// const image = new Image() +// image.onload = () => { +// resolve(image) +// } +// image.crossOrigin = 'Anonymous' +// image.onerror = (e) => { +// resolve(null!) +// console.log('failed to load image', node.imageSrc, e) +// } +// image.src = `https://cors-anywhere.herokuapp.com/${node.imageSrc!}` +// }) +// // take loaded images and convert them into an atlas map. assume the images are the square and the same size +// }) +// ) + +// const imagesLoaded = images.filter((i) => !!i) +// const atlasSize = Math.ceil(Math.sqrt(imagesLoaded.length)) +// const atlas = document.createElement('canvas') +// const resolution = 128 +// atlas.width = atlasSize * resolution +// atlas.height = atlasSize * resolution +// const ctx = atlas.getContext('2d')! +// const indicies = new Set() +// imagesLoaded.forEach((image, i) => { +// // flip image +// const x = i % atlasSize +// const y = Math.floor(i / atlasSize) +// ctx.drawImage(image!, x * resolution, y * resolution, resolution, resolution) +// indicies.add(i) +// }) +// //convert the atlas to imagedata and then a blob +// const imageData = ctx.getImageData(0, 0, atlas.width, atlas.height) + +// const texture = new DataTexture(imageData.data, atlas.width, atlas.height) +// texture.colorSpace = SRGBColorSpace +// texture.needsUpdate = true +// // texture.flipY = true +// const material = new MeshBasicMaterial({ map: texture, side: BackSide }) +// const circleGeom = new CircleGeometry(graphScale, 16).scale(1, -1, 1) // unflip y + +// const uvOffset = new Float32Array(images.length * 2) +// // set the uv offset for each image, accounting for images that are not loaded +// for (let i = 0; i < images.length; i++) { +// const isLoaded = indicies.has(i) +// if (!isLoaded) continue +// // start from top left +// const x = i % atlasSize +// const y = Math.floor(i / atlasSize) +// uvOffset[i * 2] = x / atlasSize +// uvOffset[i * 2 + 1] = y / atlasSize +// } +// circleGeom.setAttribute('uvOffset', new InstancedBufferAttribute(uvOffset, 2)) + +// material.onBeforeCompile = function (shader) { +// shader.vertexShader = shader.vertexShader.replace('void main() {', 'attribute vec2 uvOffset;\n' + 'void main() {') + +// shader.vertexShader = shader.vertexShader.replace( +// '#include ', +// '#include \n' + `vMapUv = (uv * ${1 / atlasSize}) + uvOffset;` +// ) +// } + +// const mesh = new InstancedMesh(circleGeom, material, images.length) +// mesh.frustumCulled = false +// } diff --git a/src/tools/json-schema/contentHash.ts b/src/tools/json-schema/contentHash.ts index 10ea263..3e47efc 100644 --- a/src/tools/json-schema/contentHash.ts +++ b/src/tools/json-schema/contentHash.ts @@ -1,39 +1,15 @@ +import canonicalize from 'canonicalize' import { createHash } from 'crypto' +import { JSONSchema } from 'json-schema-to-ts' +import { SHA256Hash } from '../registries/SchemaRegistry' /** - * Recursively canonicalizes a value: - * - Primitives are returned as-is. - * - Arrays are mapped through canonicalize. - * - Plain objects have their keys sorted and values canonicalized. - */ -function canonicalize(value: any): any { - if (value === null || typeof value !== 'object') { - // primitives (string, number, boolean, null) - return value - } - - if (Array.isArray(value)) { - // arrays: preserve order, but canonicalize each element - return value.map(canonicalize) - } - - // plain object: sort keys - const sortedKeys = Object.keys(value).sort() - const result: Record = {} - for (const key of sortedKeys) { - result[key] = canonicalize(value[key]) - } - return result -} - -/** - * Produces a deterministic SHA-256 hash of any JSON-like object, - * ignoring original property order. + * Produces a deterministic SHA-256 hash of any JSON-like object, canonicalising it with RFC8785 (https://datatracker.ietf.org/doc/html/rfc8785). * * @param obj - The JSON-serializable input. * @returns A hex string of the SHA-256 hash. */ -export function contentHash(obj: any): string { +export function contentHash(obj: any): SHA256Hash { // 1. Canonicalize const canon = canonicalize(obj) @@ -43,3 +19,255 @@ export function contentHash(obj: any): string { // 3. Hash return createHash('sha256').update(json, 'utf8').digest('hex') } + +/** + * Creates a copy of a JSON Schema with all metadata removed, which is then canonicalised and hashed to create a unique identifier for the structure described by the schema. + * @param schema + */ +export function contentHashJSONSchema(schema: JSONSchema): SHA256Hash { + const structural = toStructuralSchema(schema as any) + return contentHash(structural) +} + +// Keys considered annotations/metadata per JSON Schema, which should be excluded from the structural view. +const ANNOTATION_KEYS = new Set([ + 'title', + 'description', + 'examples', + 'default', + 'deprecated', + 'readOnly', + 'writeOnly', + '$comment', + '$id', + '$schema' +]) + +// Keys that affect validation/structure and must be retained. +const STRUCTURAL_KEYS = new Set([ + // Core/dollar keys + '$ref', + '$defs', + 'definitions', // legacy alias used in older drafts + + // Applicators + 'allOf', + 'anyOf', + 'oneOf', + 'not', + 'if', + 'then', + 'else', + + // Types and basic assertions + 'type', + 'const', + 'enum', + + // Numeric + 'multipleOf', + 'maximum', + 'exclusiveMaximum', + 'minimum', + 'exclusiveMinimum', + + // String + 'maxLength', + 'minLength', + 'pattern', + 'format', + 'contentEncoding', + 'contentMediaType', + 'contentSchema', + + // Array + 'items', + 'prefixItems', + 'contains', + 'minItems', + 'maxItems', + 'uniqueItems', + 'unevaluatedItems', + + // Object + 'maxProperties', + 'minProperties', + 'required', + 'properties', + 'patternProperties', + 'additionalProperties', + 'propertyNames', + 'dependentSchemas', + 'dependentRequired', + 'dependencies', // legacy alias used in older drafts + 'unevaluatedProperties' +]) + +function isObject(value: any): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function normalizeStringArray(arr: unknown): string[] | undefined { + if (!Array.isArray(arr)) return undefined + // Keep only strings, sort for determinism + return [...arr].filter((v): v is string => typeof v === 'string').sort() +} + +function sortSchemasArray(arr: any[]): any[] { + // Order of allOf/anyOf/oneOf does not affect semantics; sort deterministically by canonical form + return [...arr].sort((a, b) => { + const sa = canonicalize(toStructuralSchema(a)) ?? JSON.stringify(a) + const sb = canonicalize(toStructuralSchema(b)) ?? JSON.stringify(b) + if (sa < sb) return -1 + if (sa > sb) return 1 + return 0 + }) +} + +function mapValues(obj: Record, fn: (v: T, k: string) => R): Record { + const out: Record = {} + // Ensure deterministic key order by iterating sorted keys + for (const key of Object.keys(obj).sort()) { + out[key] = fn(obj[key], key) + } + return out +} + +// Recursively build a new schema object keeping only structural/validation-related facets. +function toStructuralSchema(schema: any): any { + // Boolean schemas are already structural by definition + if (typeof schema === 'boolean') return schema + if (!isObject(schema)) return schema + + const out: Record = {} + + // $ref short-circuits most other keywords in evaluation, but we still include it if present + if ('$ref' in schema && typeof schema.$ref === 'string') { + out.$ref = schema.$ref + // Note: Even with $ref, additional keywords MAY appear; keep them if present below. + } + + // $defs/definitions: sanitize each subschema + if (isObject(schema.$defs)) { + out.$defs = mapValues(schema.$defs, (sub) => toStructuralSchema(sub)) + } + if (isObject(schema.definitions)) { + out.definitions = mapValues(schema.definitions, (sub) => toStructuralSchema(sub)) + } + + // Combinators + for (const key of ['allOf', 'anyOf', 'oneOf'] as const) { + const val = (schema as any)[key] + if (Array.isArray(val)) out[key] = sortSchemasArray(val.map((s) => toStructuralSchema(s))) + } + if (schema.not !== undefined) out.not = toStructuralSchema(schema.not) + if (schema.if !== undefined) out.if = toStructuralSchema(schema.if) + if (schema.then !== undefined) out.then = toStructuralSchema(schema.then) + if (schema.else !== undefined) out.else = toStructuralSchema(schema.else) + + // Basic + if (schema.type !== undefined) { + if (Array.isArray(schema.type)) { + // Normalize order for determinism + out.type = [...schema.type].sort() + } else { + out.type = schema.type + } + } + if (schema.const !== undefined) out.const = schema.const + if (schema.enum !== undefined && Array.isArray(schema.enum)) { + // Sort by canonical form for determinism + out.enum = [...schema.enum].sort((a, b) => { + const sa = canonicalize(a) ?? JSON.stringify(a) + const sb = canonicalize(b) ?? JSON.stringify(b) + return sa < sb ? -1 : sa > sb ? 1 : 0 + }) + } + + // Numeric + for (const key of ['multipleOf', 'maximum', 'exclusiveMaximum', 'minimum', 'exclusiveMinimum'] as const) { + if (schema[key] !== undefined) out[key] = schema[key] + } + + // String + for (const key of ['maxLength', 'minLength', 'pattern', 'format', 'contentEncoding', 'contentMediaType'] as const) { + if (schema[key] !== undefined) out[key] = schema[key] + } + if (schema.contentSchema !== undefined) out.contentSchema = toStructuralSchema(schema.contentSchema) + + // Array + if (schema.items !== undefined) { + if (Array.isArray(schema.items)) { + // Tuple validation – order is significant, so do NOT sort + out.items = schema.items.map((s: any) => toStructuralSchema(s)) + } else { + out.items = toStructuralSchema(schema.items) + } + } + if (Array.isArray(schema.prefixItems)) { + // Order matters – do NOT sort + out.prefixItems = schema.prefixItems.map((s: any) => toStructuralSchema(s)) + } + if (schema.contains !== undefined) out.contains = toStructuralSchema(schema.contains) + for (const key of ['minItems', 'maxItems', 'uniqueItems'] as const) { + if (schema[key] !== undefined) out[key] = schema[key] + } + if (schema.unevaluatedItems !== undefined) { + out.unevaluatedItems = + typeof schema.unevaluatedItems === 'boolean' + ? schema.unevaluatedItems + : toStructuralSchema(schema.unevaluatedItems) + } + + // Object + for (const key of ['maxProperties', 'minProperties'] as const) { + if (schema[key] !== undefined) out[key] = schema[key] + } + const req = normalizeStringArray(schema.required) + if (req && req.length) out.required = req + if (isObject(schema.properties)) { + out.properties = mapValues(schema.properties, (sub) => toStructuralSchema(sub)) + } + if (isObject(schema.patternProperties)) { + out.patternProperties = mapValues(schema.patternProperties, (sub) => toStructuralSchema(sub)) + } + if (schema.additionalProperties !== undefined) { + out.additionalProperties = + typeof schema.additionalProperties === 'boolean' + ? schema.additionalProperties + : toStructuralSchema(schema.additionalProperties) + } + if (schema.propertyNames !== undefined) { + out.propertyNames = toStructuralSchema(schema.propertyNames) + } + if (isObject(schema.dependentSchemas)) { + out.dependentSchemas = mapValues(schema.dependentSchemas, (sub) => toStructuralSchema(sub)) + } + if (isObject(schema.dependentRequired)) { + out.dependentRequired = mapValues(schema.dependentRequired, (arr) => normalizeStringArray(arr as any) ?? []) + } + if (isObject(schema.dependencies)) { + // Legacy: values can be schema or array of strings + out.dependencies = mapValues(schema.dependencies as Record, (val) => { + if (Array.isArray(val)) return normalizeStringArray(val) ?? [] + return toStructuralSchema(val) + }) + } + if (schema.unevaluatedProperties !== undefined) { + out.unevaluatedProperties = + typeof schema.unevaluatedProperties === 'boolean' + ? schema.unevaluatedProperties + : toStructuralSchema(schema.unevaluatedProperties) + } + + // Finally, ensure we didn't accidentally copy annotations + for (const k in schema) { + if (!STRUCTURAL_KEYS.has(k) && !ANNOTATION_KEYS.has(k)) { + // Unknown keys: conservatively include if they are objects/arrays that look like schemas affecting validation. + // To avoid metadata leakage, only include recognized structural keywords. + // So we intentionally skip unknown keys. + } + } + + return out +} diff --git a/src/tools/json-schema/generateJsonSchema.ts b/src/tools/json-schema/generateJsonSchema.ts index 67bfaad..a4beb00 100644 --- a/src/tools/json-schema/generateJsonSchema.ts +++ b/src/tools/json-schema/generateJsonSchema.ts @@ -1,66 +1,129 @@ import type { JSONSchemaType } from './JSONSchema' +// RFC 3339 date-time as referenced by JSON Schema "date-time" format (RFC 3339, section 5.6) +// Example: 2020-12-31T23:59:59Z, 2020-12-31T23:59:59.123Z, 2020-12-31T23:59:59+01:00 +// This checks structural conformance; it does not validate calendar edge cases like Feb 30th. +const RFC3339_DATE_TIME = + /^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):([0-5]\d):([0-5]\d)(?:\.(\d+))?(Z|[+\-]([01]\d|2[0-3]):[0-5]\d)$/ + +function isRFC3339DateTime(value: string): boolean { + return RFC3339_DATE_TIME.test(value) +} + /** - * Recursively infers a JSON Schema for a given value. - * @param value - The value to inspect. - * @returns A JSONSchema object compliant with the JSON Schema specification. + * Infer a JSON Schema that accepts all of the provided values. + * This merges heterogeneous types using anyOf and computes object.required. */ -function inferJsonSchemaForValue(value: any): JSONSchemaType { - if (value === null || value === undefined) { - // When the value is null or undefined, we return a schema that accepts null. +function inferSchemaFromValues(values: any[]): JSONSchemaType { + // Classify values + const nonUndef = values.filter((v) => v !== undefined) + if (nonUndef.length === 0) { + // If everything was undefined, allow null return { type: 'null', nullable: true } } - if (typeof value === 'number') { - return { type: 'number' } + + const hasNull = nonUndef.some((v) => v === null) + const scalars = nonUndef.filter((v) => v !== null && typeof v !== 'object') + const arrays = nonUndef.filter((v) => Array.isArray(v)) as any[] + const objects = nonUndef.filter((v) => v !== null && typeof v === 'object' && !Array.isArray(v)) as Record< + string, + any + >[] + + // Primitive types + const boolCount = scalars.filter((v) => typeof v === 'boolean').length + const numVals = scalars.filter((v) => typeof v === 'number') as number[] + const strVals = scalars.filter((v) => typeof v === 'string') as string[] + + const variants: JSONSchemaType[] = [] + + // booleans + if (boolCount > 0) variants.push({ type: 'boolean' }) + + // numbers: prefer integer if all observed numbers are integers + if (numVals.length > 0) { + const allIntegers = numVals.every((n) => Number.isInteger(n)) + variants.push({ type: allIntegers ? 'integer' : 'number' } as JSONSchemaType) } - if (typeof value === 'boolean') { - return { type: 'boolean' } + + // strings: keep format only if every string matches it; otherwise drop format + if (strVals.length > 0) { + const allDateTime = strVals.length > 0 && strVals.every((s) => isRFC3339DateTime(s)) + variants.push( + allDateTime + ? ({ type: 'string', format: 'date-time' } as JSONSchemaType) + : ({ type: 'string' } as JSONSchemaType) + ) } - if (typeof value === 'string') { - // Check if the string can be parsed as a date. - const timestamp = Date.parse(value) - if (!isNaN(timestamp)) { - return { type: 'string', format: 'date-time' } + + // arrays: merge items across all array elements + if (arrays.length > 0) { + const allItems: any[] = [] + for (const arr of arrays as any[][]) { + for (const item of arr) allItems.push(item) } - return { type: 'string' } + const itemsSchema: JSONSchemaType = + allItems.length > 0 ? inferSchemaFromValues(allItems) : ({} as JSONSchemaType) + variants.push({ type: 'array', items: itemsSchema } as JSONSchemaType) } - if (Array.isArray(value)) { - // For arrays, attempt to infer the schema from the first non-null element. - const sample = value.find((item) => item !== null && item !== undefined) - /** @todo we should include 'optional' as a field here for all properties only in some entries of the array */ - if (sample !== undefined) { - return { type: 'array', items: inferJsonSchemaForValue(sample) } - } else { - // If no sample is found, allow any items. - return { type: 'array', items: {} as JSONSchemaType } + + // objects: merge properties and required keys across all objects + if (objects.length > 0) { + const keySet = new Set() + const presentCount = new Map() + const valueBuckets: Record = {} + + for (const obj of objects) { + const keys = Object.keys(obj) + for (const k of keys) { + keySet.add(k) + presentCount.set(k, (presentCount.get(k) ?? 0) + 1) + if (!valueBuckets[k]) valueBuckets[k] = [] + valueBuckets[k].push(obj[k]) + } } - } - if (typeof value === 'object') { + const properties: { [key: string]: JSONSchemaType } = {} - Object.keys(value).forEach((key) => { - properties[key] = inferJsonSchemaForValue(value[key]) - }) - return { type: 'object', properties } as JSONSchemaType + for (const k of keySet) { + properties[k] = inferSchemaFromValues(valueBuckets[k] ?? []) + } + const required: string[] = [] + for (const k of keySet) { + if ((presentCount.get(k) ?? 0) === objects.length) required.push(k) + } + + const objectSchema: JSONSchemaType = { + type: 'object', + properties, + ...(required.length > 0 ? { required } : {}) + } as unknown as JSONSchemaType + variants.push(objectSchema) } - // Fallback: return string type. - return { type: 'string' } + + // Build final union, adding null when present + const nullSchema = hasNull ? ({ type: 'null', nullable: true } as JSONSchemaType) : undefined + + if (variants.length === 0) { + // Only nulls present + return nullSchema ?? ({ type: 'string' } as JSONSchemaType) + } + + if (variants.length === 1 && !nullSchema) return variants[0] + if (variants.length === 1 && nullSchema) return { anyOf: [variants[0], nullSchema] } as JSONSchemaType + + // Multiple variants + return { anyOf: [...variants, ...(nullSchema ? [nullSchema] : [])] } as JSONSchemaType } /** - * Generates a JSON Schema for the provided raw data. - * If the raw data is not an array, it wraps it in one. - * The returned schema describes an array of items. - * @param rawData - The raw data fetched from the endpoint. - * @returns A JSONSchema object representing the data structure. + * Generates a JSON Schema for the provided raw data, merging all observed values. + * - If rawData is an array, returns an array schema whose items accept all item variants. + * - If rawData is a single value/object, returns a schema for that value. */ export function generateJsonSchema(rawData: any): JSONSchemaType { if (Array.isArray(rawData)) { - // Infer the schema from the first item (assuming homogeneity). - const itemSchema = inferJsonSchemaForValue(rawData[0]) - return { - type: 'array', - items: itemSchema - } + const itemSchema = rawData.length > 0 ? inferSchemaFromValues(rawData) : ({} as JSONSchemaType) + return { type: 'array', items: itemSchema } as JSONSchemaType } - return inferJsonSchemaForValue(rawData) + return inferSchemaFromValues([rawData]) } diff --git a/src/tools/llm/useLLM.ts b/src/tools/llm/useLLM.ts index f2dac85..7edbe5e 100644 --- a/src/tools/llm/useLLM.ts +++ b/src/tools/llm/useLLM.ts @@ -200,11 +200,9 @@ function extractJavascript(text: string): unknown { /** * Call the LLM with a prompt and JSON schema for structured output */ -async function callLLM(engine: MLCEngineInterface, options: LLMCallOptions): Promise> { +async function callMLC(engine: MLCEngineInterface, options: LLMCallOptions): Promise> { const { prompt, temperature = 0.7, maxTokens = 1000 } = options - console.log('calling prompt:', prompt) - try { const response = await engine.chat.completions.create({ messages: [ @@ -219,7 +217,6 @@ async function callLLM(engine: MLCEngineInterface, options: LLMCall const rawResponse = response.choices[0]?.message?.content || '' - console.log('rawResponse:', rawResponse) if (!rawResponse) { throw new Error('Empty response from LLM') } @@ -360,6 +357,7 @@ async function callRemoteLLM( } return { data: parsedData as T, rawResponse, isValid: true } } + return { data: rawResponse as T, rawResponse, isValid: true } } @@ -422,7 +420,7 @@ export function useLLM(options: LLMInitOptions & { apiKey?: string; ollamaUrl?: if (!selectedModel) throw new Error('No model selected') if (selectedModel.provider === 'mlc') { if (!llm.engine.value) throw new Error('LLM not initialized') - return callLLM(llm.engine.value as MLCEngineInterface, options) + return callMLC(llm.engine.value as MLCEngineInterface, options) } else { // For remote models, require apiKey or ollamaUrl as needed if (selectedModel.provider === 'ollama') { @@ -440,3 +438,32 @@ export function useLLM(options: LLMInitOptions & { apiKey?: string; ollamaUrl?: progress } } + +export const callLLM = async ( + callOptions: LLMCallOptions, + options: LLMInitOptions & { apiKey?: string; ollamaUrl?: string } = {} +) => { + const { modelId } = options + const selectedModel = CODING_MODELS.find((m) => m.id === modelId) + + if (!selectedModel) return + + /** @todo handle multiple promises */ + if (!llm.engine.value && selectedModel.provider === 'mlc') { + const llmInstance = await initializeEngine() + llm.engine.set(llmInstance) + llm.currentModelId.set(modelId || 'Llama-3.2-3B-Instruct-q4f32_1-MLC') + llm.initializing.set(false) + } + + if (selectedModel.provider === 'mlc') { + return callMLC(llm.engine.value as MLCEngineInterface, callOptions) + } + + if (selectedModel.provider === 'ollama') { + return callRemoteLLM(selectedModel, callOptions, '', options.ollamaUrl) + } + + if (!options.apiKey) throw new Error('API key required for remote LLM') + return callRemoteLLM(selectedModel, callOptions, options.apiKey, options.ollamaUrl) +} diff --git a/src/tools/pipeline/EditorContext.tsx b/src/tools/pipeline/EditorContext.tsx new file mode 100644 index 0000000..72fdf40 --- /dev/null +++ b/src/tools/pipeline/EditorContext.tsx @@ -0,0 +1,13 @@ +import React from 'react' + +export type ToolLite = { + hash: string + label: string +} + +export type EditorContextValue = { + tools: ToolLite[] + updateNodeConfig: (id: string, config: any) => void +} + +export const EditorContext = React.createContext({ tools: [], updateNodeConfig: () => {} }) diff --git a/src/tools/pipeline/Node.tsx b/src/tools/pipeline/Node.tsx new file mode 100644 index 0000000..f63660d --- /dev/null +++ b/src/tools/pipeline/Node.tsx @@ -0,0 +1,109 @@ +import React, { useContext, useMemo } from 'react' +import { Handle, NodeProps, Position } from 'reactflow' +import type { JSONSchemaType } from '../json-schema/JSONSchema' +import { EditorContext } from './EditorContext' +import { SchemaForm } from './SchemaForm' + +const inputPasteSchema: JSONSchemaType<{ text: string; format: 'json' | 'csv' }> = { + type: 'object', + properties: { + text: { type: 'string', title: 'Data' }, + format: { type: 'string', enum: ['json', 'csv'], default: 'json', title: 'Format' } + }, + required: ['text'] +} + +const transformSchema: JSONSchemaType<{ toolHash: string | null }> = { + type: 'object', + properties: { toolHash: { type: 'string', title: 'Tool Hash' } }, + required: [] +} + +const inputUrlSchema: JSONSchemaType<{ url: string; schemaHash?: string }> = { + type: 'object', + properties: { + url: { type: 'string', title: 'URL' }, + schemaHash: { type: 'string', title: 'Schema Hash', nullable: true } + }, + required: ['url'] +} + +export function DbNode({ id, data, selected }: NodeProps) { + const { tools: toolList, updateNodeConfig } = useContext(EditorContext) + + const title = useMemo(() => { + const t = data?.type as string + if (!t) return 'Node' + const [group, name] = t.split('.') + return `${group}: ${name}` + }, [data?.type]) + + const renderControls = () => { + switch (data?.type) { + case 'input.paste': + return ( + updateNodeConfig(id, v)} + /> + ) + case 'input.url': + return ( + updateNodeConfig(id, v)} + /> + ) + case 'xform.js': + case 'xform.filter': + case 'xform.merge': + case 'xform.group': + case 'xform.slice': + case 'xform.sort': + case 'xform.rename': + case 'xform.geocode': + case 'xform.color': + return ( +
    + updateNodeConfig(id, v)} + /> +
    + + +
    +
    + ) + default: + return
    No controls
    + } + } + + return ( +
    +
    +
    {title}
    +
    +
    {renderControls()}
    + + +
    + ) +} diff --git a/src/tools/pipeline/PipelineEditor.tsx b/src/tools/pipeline/PipelineEditor.tsx new file mode 100644 index 0000000..332b80f --- /dev/null +++ b/src/tools/pipeline/PipelineEditor.tsx @@ -0,0 +1,471 @@ +import { getState } from '@ir-engine/hyperflux' +import React, { useCallback, useMemo, useRef, useState } from 'react' +import { Rnd } from 'react-rnd' +import ReactFlow, { + applyEdgeChanges, + applyNodeChanges, + Background, + Connection, + ConnectionMode, + Controls, + Edge, + EdgeChange, + MiniMap, + Node, + NodeChange, + OnConnectEnd, + OnConnectStart, + ReactFlowProvider, + useReactFlow +} from 'reactflow' +import 'reactflow/dist/style.css' +import { SchemaRegistry } from '../registries/SchemaRegistry' +import { ToolRegistry } from '../registries/ToolRegistry' +import { EditorContext, ToolLite } from './EditorContext' +import { DbNode } from './Node' + +export type PipelineGraph = { nodes: Node[]; edges: Edge[] } + +// Internal canvas component (defined at module scope to keep identity stable across renders) +type DnDFlowProps = { + graph: PipelineGraph + nodeTypes: Record + onNodesChange: (changes: NodeChange[]) => void + onEdgesChange: (changes: EdgeChange[]) => void + onConnect: (connection: Connection) => void + onSelected: (id: string | null) => void + onAddToolNode: (position: { x: number; y: number }, toolHash: string) => void + onOpenConnectMenu: (payload: { + x: number + y: number + flowPos: { x: number; y: number } + from: { nodeId: string; handleType: 'source' | 'target' } + }) => void + pendingConnectRef: React.MutableRefObject<{ nodeId: string; handleType: 'source' | 'target' } | null> +} + +const DnDFlow: React.FC = ({ + graph, + nodeTypes, + onNodesChange, + onEdgesChange, + onConnect, + onSelected, + onAddToolNode, + onOpenConnectMenu, + pendingConnectRef +}) => { + const rf = useReactFlow() + + const onDragOver = useCallback((event: React.DragEvent) => { + // Only indicate drop when dragging our reactflow payload + const hasRF = Array.from(event.dataTransfer.types || []).includes('application/reactflow') + if (!hasRF) return + event.preventDefault() + event.dataTransfer.dropEffect = 'move' + }, []) + + const onDrop = useCallback( + (event: React.DragEvent) => { + event.preventDefault() + const raw = event.dataTransfer.getData('application/reactflow') + if (!raw) return + let payload: any + try { + payload = JSON.parse(raw) + } catch { + return + } + const position = rf.screenToFlowPosition({ x: event.clientX, y: event.clientY }) + if (payload.kind === 'tool' && payload.hash) { + onAddToolNode(position, payload.hash) + } + }, + [onAddToolNode, rf] + ) + + const onConnectStart = useCallback((_, params) => { + if (!params.nodeId || !params.handleType) return + pendingConnectRef.current = { nodeId: params.nodeId, handleType: params.handleType } + }, []) + + const onConnectEnd = useCallback( + (event) => { + const isPane = (event.target as HTMLElement)?.classList?.contains('react-flow__pane') + if (!isPane || !pendingConnectRef.current) return + const flowPos = rf.screenToFlowPosition({ x: (event as MouseEvent).clientX, y: (event as MouseEvent).clientY }) + onOpenConnectMenu({ + x: (event as MouseEvent).clientX, + y: (event as MouseEvent).clientY, + flowPos, + from: pendingConnectRef.current + }) + pendingConnectRef.current = null + }, + [onOpenConnectMenu, rf] + ) + + return ( + onSelected(sel.nodes[0]?.id ?? null)} + nodeTypes={nodeTypes} + fitView + onDrop={onDrop} + onDragOver={onDragOver} + onConnectStart={onConnectStart} + onConnectEnd={onConnectEnd} + connectionMode={ConnectionMode.Loose} + > + + + + + ) +} + +type Props = { + graph: PipelineGraph + onChange: (graph: PipelineGraph) => void + onRun: () => void + onSave: (graph: PipelineGraph) => void + tools?: ToolLite[] + nodeOutputs?: Record +} + +export const PipelineEditor: React.FC = ({ graph, onChange, onRun, onSave, tools = [], nodeOutputs }) => { + const [showLeft, setShowLeft] = useState(true) + const [showRight, setShowRight] = useState(true) + const [selectedNodeId, setSelectedNodeId] = useState(null) + const [connectMenu, setConnectMenu] = useState<{ + x: number + y: number + flowPos: { x: number; y: number } + from: { nodeId: string; handleType: 'source' | 'target' } + } | null>(null) + const pendingConnectRef = useRef<{ nodeId: string; handleType: 'source' | 'target' } | null>(null) + + const nodeTypes = useMemo(() => ({ db: DbNode }), []) + + const onNodesChange = useCallback( + (changes: NodeChange[]) => { + onChange({ nodes: applyNodeChanges(changes, graph.nodes), edges: graph.edges }) + }, + [graph.nodes, graph.edges, onChange] + ) + const onEdgesChange = useCallback( + (changes: EdgeChange[]) => { + onChange({ nodes: graph.nodes, edges: applyEdgeChanges(changes, graph.edges) }) + }, + [graph.nodes, graph.edges, onChange] + ) + const onConnect = useCallback( + (connection: Connection) => { + const id = `e_${crypto.randomUUID()}` + onChange({ nodes: graph.nodes, edges: [...graph.edges, { id, ...connection } as any] }) + }, + [graph.nodes, graph.edges, onChange] + ) + + const updateNodeConfig = useCallback( + (id: string, config: any) => { + const nodes = graph.nodes.map((n) => (n.id === id ? { ...n, data: { ...(n.data || {}), config } } : n)) + onChange({ nodes, edges: graph.edges }) + }, + [graph.nodes, graph.edges, onChange] + ) + + // helper to add tool node at position + const addToolNodeAt = useCallback( + (position: { x: number; y: number }, hash: string) => { + const id = `n_${crypto.randomUUID()}` + const newNode: Node = { + id, + type: 'db', + position, + data: { type: 'tool', config: { toolHash: hash } } + } as any + onChange({ nodes: [...graph.nodes, newNode], edges: graph.edges }) + setSelectedNodeId(id) + }, + [graph.nodes, graph.edges, onChange] + ) + + // Helper to create node and edge from connect menu action + const createNodeFromMenu = useCallback( + (kind: 'tool' | 'input' | 'output', toolHash?: string) => { + const menu = connectMenu + if (!menu) return + const id = `n_${crypto.randomUUID()}` + let data: any + if (kind === 'tool') data = { type: 'tool', config: { toolHash: toolHash || '' } } + else if (kind === 'input') data = { type: 'input.url', config: { url: '', schemaHash: '' } } + else data = { type: 'viz.table', config: {} } + + const newNode: Node = { id, type: 'db', position: menu.flowPos, data } as any + + // Wire the edge depending on starting handle + const from = menu.from as { nodeId: string; handleType: 'source' | 'target' } + const edge: Edge = + from.handleType === 'source' + ? ({ id: `e_${crypto.randomUUID()}`, source: from.nodeId, target: id } as any) + : ({ id: `e_${crypto.randomUUID()}`, source: id, target: from.nodeId } as any) + + onChange({ nodes: [...graph.nodes, newNode], edges: [...graph.edges, edge] }) + setSelectedNodeId(id) + setConnectMenu(null) + }, + [connectMenu, graph.nodes, graph.edges, onChange] + ) + + return ( +
    +
    + + + setConnectMenu(p)} + pendingConnectRef={pendingConnectRef} + /> + + +
    + + {showLeft && ( + +
    +
    +
    Library
    + +
    + + {tools.length === 0 &&
    No tools available
    } +
    + {tools.map((tool) => ( +
    + e.dataTransfer.setData('application/reactflow', JSON.stringify({ kind: 'tool', hash: tool.hash })) + } + title="Drag onto canvas" + > + {tool.label} + ({tool.hash.slice(0, 8)}) +
    + ))} +
    +
    +     + )} + + {showRight && ( + +
    +
    +
    Inspector
    + +
    + {selectedNodeId ? ( + n.id === selectedNodeId) || null} outputs={nodeOutputs} /> + ) : ( +
    Select a node to edit…
    + )} +
    +     + )} + + {connectMenu && ( +
    +
    +
    Add node
    +
    + {connectMenu.from.handleType === 'target' && ( + + )} + {connectMenu.from.handleType === 'source' && ( + + )} +
    + +
    + + +
    +
    +
    + +
    +
    +
    +
    + )} + +
    + + +
    +
    + ) +} + +// Inspector component for right panel +const Inspector: React.FC<{ node: Node | null; outputs?: Record }> = ({ node, outputs }) => { + if (!node) return null + const data: any = node.data || {} + const type = data.type as string + + if (type === 'tool' || (typeof type === 'string' && type.startsWith('xform'))) { + const toolHash = data?.config?.toolHash + const tool = toolHash ? getState(ToolRegistry).tools[toolHash] : null + return ( +
    +
    Tool
    +
    +
    Hash
    +
    {toolHash || '—'}
    +
    + {tool ? ( + <> +
    +
    Label
    +
    {tool.label}
    +
    +
    +
    Input schema
    +
    + {JSON.stringify(tool.input, null, 2)} +
    +
    +
    +
    Output schema
    +
    + {JSON.stringify(tool.output, null, 2)} +
    +
    +
    +
    Transformer
    + {typeof tool.transformation === 'string' ? ( + 
    +                  {tool.transformation}
+
    + ) : ( +
    + {JSON.stringify(tool.transformation, null, 2)} +
    + )} +
    + + ) : ( +
    Select a tool for this node.
    + )} +
    + ) + } + + if (typeof type === 'string' && type.startsWith('input')) { + const url = data?.config?.url || '—' + const schemaHash = data?.config?.schemaHash || '' + const schema = schemaHash ? getState(SchemaRegistry).schemas[schemaHash] : null + return ( +
    +
    Input
    +
    +
    URL
    +
    {url}
    +
    +
    +
    Schema
    + {schema ? ( +
    + {JSON.stringify(schema.schema, null, 2)} +
    + ) : ( +
    No schema
    + )} +
    +
    + ) + } + + // Output / Viz node + if (typeof type === 'string' && (type.startsWith('viz') || type.startsWith('output'))) { + const outputHash = data?.config?.outputHash + const schema = outputHash ? getState(SchemaRegistry).schemas[outputHash] : null + const current = outputs?.[node.id] + return ( +
    +
    Output
    +
    +
    Schema
    + {schema ? ( +
    + {JSON.stringify(schema.schema, null, 2)} +
    + ) : ( +
    No schema
    + )} +
    +
    +
    Current data
    + {current ? ( +
    + {JSON.stringify(current, null, 2)} +
    + ) : ( +
    No data available.
    + )} +
    +
    + ) + } + + return
    No inspector available.
    +} diff --git a/src/tools/pipeline/SchemaForm.tsx b/src/tools/pipeline/SchemaForm.tsx new file mode 100644 index 0000000..0678079 --- /dev/null +++ b/src/tools/pipeline/SchemaForm.tsx @@ -0,0 +1,86 @@ +import React from 'react' +import type { JSONSchemaType } from '../json-schema/JSONSchema' + +type Props = { + schema: JSONSchemaType + value: any + onChange: (val: any) => void +} + +export function SchemaForm({ schema, value, onChange }: Props) { + if (!schema || schema.type !== 'object' || !schema.properties) return null + const props = schema.properties as Record + + const set = (key: string, v: any) => { + onChange({ ...(value as any), [key]: v }) + } + + return ( +
    + {Object.entries(props).map(([key, prop]) => { + const title = prop.title || key + const req = (schema.required as string[] | undefined)?.includes(key) + if (prop.enum) { + return ( + + ) + } + switch (prop.type) { + case 'string': + return ( + + ) + case 'number': + case 'integer': + return ( + + ) + case 'boolean': + return ( + + ) + default: + return null + } + })} +
    + ) +} diff --git a/src/tools/pipeline/graphConvert.ts b/src/tools/pipeline/graphConvert.ts new file mode 100644 index 0000000..c1dee55 --- /dev/null +++ b/src/tools/pipeline/graphConvert.ts @@ -0,0 +1,69 @@ +import type { Edge, Node } from 'reactflow' +import type { PipelineSpec, PipelineStage } from './model' + +export type DbNodeData = { type: string; config?: any } + +export const graphToPipeline = (nodes: Node[], edges: Edge[]): PipelineSpec => { + const idToIndex = new Map() + const stages: PipelineStage[] = nodes.map((n, idx) => { + idToIndex.set(n.id, idx) + const { type, config } = (n.data as DbNodeData) || { type: 'input', config: {} } + if (type.startsWith('input')) { + return { type: 'input', params: config, next: [] } + } + if (type.startsWith('xform') || type === 'tool') { + return { type: 'tool', toolHash: (config as any)?.toolHash, params: config, next: [] } + } + return { type: 'output', params: config, next: [] } + }) + edges.forEach((e) => { + const s = idToIndex.get(e.source) + const t = idToIndex.get(e.target) + if (s == null || t == null) return + stages[s].next.push(t) + }) + return { stages } +} + +export const pipelineToGraph = (pipeline: PipelineSpec) => { + const nodes: Node[] = [] + const edges: Edge[] = [] + const xGap = 260 + const yGap = 140 + const incomingCounts = new Array(pipeline.stages.length).fill(0) + pipeline.stages.forEach((s) => s.next.forEach((n) => (incomingCounts[n] += 1))) + const startIndices = pipeline.stages.map((_, i) => i).filter((i) => incomingCounts[i] === 0) + const visited = new Set() + const layer: number[] = [...startIndices] + const layers: number[][] = [] + while (layer.length) { + layers.push([...layer]) + const nextLayer: number[] = [] + for (const i of layer) { + visited.add(i) + for (const j of pipeline.stages[i].next) if (!visited.has(j)) nextLayer.push(j) + } + layer.length = 0 + nextLayer.forEach((n) => layer.push(n)) + } + pipeline.stages.forEach((_, i) => { + if (!visited.has(i)) layers.push([i]) + }) + const idxToId: string[] = [] + layers.forEach((list, yi) => { + list.forEach((idx, xi) => { + const s = pipeline.stages[idx] + const id = `n_${idx}` + idxToId[idx] = id + const data: DbNodeData = + s.type === 'input' + ? { type: 'input.url', config: s.params || {} } + : s.type === 'tool' + ? { type: 'xform.js', config: { ...(s.params || {}), toolHash: (s as any).toolHash } } + : { type: 'viz.table', config: s.params || {} } + nodes.push({ id, type: 'db', data, position: { x: 80 + xi * xGap, y: 100 + yi * yGap } } as any) + }) + }) + pipeline.stages.forEach((s, i) => s.next.forEach((t) => edges.push({ id: `e_${i}_${t}`, source: idxToId[i], target: idxToId[t] } as any))) + return { nodes, edges } +} diff --git a/src/tools/pipeline/model.ts b/src/tools/pipeline/model.ts new file mode 100644 index 0000000..8811564 --- /dev/null +++ b/src/tools/pipeline/model.ts @@ -0,0 +1,22 @@ +export type PipelineStage = + | { + type: 'input' + label?: string + params?: { url?: string; schemaHash?: string; text?: string; format?: 'json' | 'csv'; data?: any } + next: number[] + } + | { + type: 'tool' + label?: string + toolHash: string + params?: Record + next: number[] + } + | { + type: 'output' + label?: string + params?: { outputHash?: string | null } + next: number[] + } + +export type PipelineSpec = { stages: PipelineStage[] } diff --git a/src/tools/pipeline/runPipelineSpec.ts b/src/tools/pipeline/runPipelineSpec.ts new file mode 100644 index 0000000..4f5b36b --- /dev/null +++ b/src/tools/pipeline/runPipelineSpec.ts @@ -0,0 +1,67 @@ +import { getState } from '@ir-engine/hyperflux' +import { TargetRegistry } from '../registries/TargetRegistry' +import { ToolRegistry } from '../registries/ToolRegistry' +import { PipelineSpec } from './model' + +export async function runPipelineSpec(spec: PipelineSpec): Promise { + const { stages } = spec + const incoming: Record = {} + const outgoing: Record = {} + stages.forEach((_, i) => { + incoming[i] = [] + outgoing[i] = [] + }) + stages.forEach((s, i) => s.next.forEach((t) => outgoing[i].push(t))) + stages.forEach((_, t) => stages.forEach((s, i) => s.next.includes(t) && incoming[t].push(i))) + + const evaluated = new Set() + const stageData: Record = {} + + const evalStage = async (i: number) => { + const s = stages[i] + const ins = (incoming[i] || []).map((sid) => stageData[sid]).filter((v) => v !== undefined) + try { + if (s.type === 'input') { + if (s.params?.data !== undefined) stageData[i] = s.params.data + else if (s.params?.text) { + if (s.params.format === 'csv') { + const [head, ...rows] = s.params.text.trim().split(/\r?\n/) + const headers = head.split(',') + stageData[i] = rows.map((r: string) => { + const vals = r.split(',') + return Object.fromEntries(headers.map((h, idx) => [h, vals[idx]])) + }) + } else { + stageData[i] = JSON.parse(s.params.text) + } + } else { + stageData[i] = null + } + } else if (s.type === 'tool') { + const toolHash = s.toolHash + const input = ins[0] + stageData[i] = toolHash ? await ToolRegistry.run(toolHash as any, input) : input + } else if (s.type === 'output') { + const outputHash = s.params?.outputHash + if (!outputHash) return + const targetGraph = getState(TargetRegistry)[outputHash] + targetGraph.deserialize(stageData) + } + } catch (e) { + console.error('Pipeline stage failed', i, e) + stageData[i] = undefined + } + evaluated.add(i) + } + + for (let pass = 0; pass < stages.length; pass++) { + for (let i = 0; i < stages.length; i++) { + if (evaluated.has(i)) continue + const inc = incoming[i] || [] + if (inc.length === 0 || inc.every((sid) => evaluated.has(sid))) { + // eslint-disable-next-line no-await-in-loop + await evalStage(i) + } + } + } +} diff --git a/src/tools/registries/PipelineRegistry.tsx b/src/tools/registries/PipelineRegistry.tsx new file mode 100644 index 0000000..81b350d --- /dev/null +++ b/src/tools/registries/PipelineRegistry.tsx @@ -0,0 +1,106 @@ +import { defineState, getMutableState, NO_PROXY, none, useMutableState } from '@ir-engine/hyperflux' +import React, { useEffect } from 'react' +import { P2P_API } from '../../api/CRUD' +import { contentHash } from '../json-schema/contentHash' +import { graphToPipeline } from '../pipeline/graphConvert' +import type { PipelineSpec } from '../pipeline/model' + +export type Pipeline = { + hash: string + label: string + description: string + graph: PipelineSpec +} + +export const PIPELINE_PREDICATE = 'conjure://pipeline' + +export const PipelineRegistry = defineState({ + name: 'hexafield.conjure.PipelineRegistry', + initial: { pipelines: {} as Record }, + + register: (pipeline: Omit) => { + const hash = contentHash({ + label: pipeline.label, + description: pipeline.description, + graph: pipeline.graph + }) + const payload: Pipeline = { ...pipeline, hash } + getMutableState(PipelineRegistry).pipelines[hash].set(payload) + return hash + }, + + forget: (hash: string) => { + getMutableState(PipelineRegistry).pipelines[hash].set(none) + if (!P2P_API.client) return + P2P_API.client.delete({ source: hash, predicate: PIPELINE_PREDICATE }).then(async () => { + console.log('deleted pipeline:', hash) + }) + }, + + reactor: () => { + const pipelineState = useMutableState(PipelineRegistry).pipelines + const apiReady = useMutableState(P2P_API).ready.value + + useEffect(() => { + if (!apiReady) return + P2P_API.client.find({ predicate: PIPELINE_PREDICATE }).then((sources) => { + sources.forEach(async (source) => { + P2P_API.client + .get({ source, predicate: PIPELINE_PREDICATE }) + .then(async (response: object) => { + if (!response) return + const { label, description, graph } = response as any + let spec: PipelineSpec + if (graph && Array.isArray(graph.stages)) { + spec = graph as PipelineSpec + } else if (graph && Array.isArray(graph.nodes) && Array.isArray(graph.edges)) { + // migrate legacy graph to spec + spec = graphToPipeline(graph.nodes, graph.edges) + } else { + spec = { stages: [] } + } + PipelineRegistry.register({ label, description, graph: spec }) + }) + .catch((e) => { + console.error('Failed to retrieve pipeline:', e) + }) + }) + }) + }, [apiReady]) + + if (!apiReady) return null + + return ( + <> + {pipelineState.keys.map((key) => ( + + ))} + + ) + } +}) + +const SyncPipeline = ({ hash }: { hash: string }) => { + const pipeline = useMutableState(PipelineRegistry).pipelines[hash].get(NO_PROXY) + + useEffect(() => { + P2P_API.client.has({ source: hash, predicate: PIPELINE_PREDICATE }).then(async (exists) => { + if (exists) return + P2P_API.client + .create({ + predicate: PIPELINE_PREDICATE, + source: hash, + target: { + label: pipeline.label, + description: pipeline.description, + graph: pipeline.graph + } + }) + .catch((e) => { + console.error('Failed to create pipeline:', e) + }) + }) + }, [JSON.stringify(pipeline)]) + + return null +} diff --git a/src/tools/registries/SchemaRegistry.tsx b/src/tools/registries/SchemaRegistry.tsx index 17095c7..5b39e4e 100644 --- a/src/tools/registries/SchemaRegistry.tsx +++ b/src/tools/registries/SchemaRegistry.tsx @@ -1,6 +1,15 @@ -import { defineState, getMutableState, none, syncStateWithLocalStorage } from '@ir-engine/hyperflux' +import { compileSchema, SchemaNode } from 'json-schema-library' +import React, { useEffect } from 'react' + +import { defineState, getMutableState, getState, NO_PROXY, none, useMutableState } from '@ir-engine/hyperflux' + +import { P2P_API } from '../../api/CRUD' import { JSONSchemaType } from '../json-schema/JSONSchema' -import { contentHash } from '../json-schema/contentHash' +import { contentHashJSONSchema } from '../json-schema/contentHash' +import { generateJsonSchema } from '../json-schema/generateJsonSchema' +import { registerKnownSchemas } from '../schemas/KnownSchemas' + +export const SCHEMA_PREDICATE = 'conjure://schema' export type SHA256Hash = string @@ -11,12 +20,16 @@ export type SchemaType = { schema: JSONSchemaType } +const validators = {} as Record + export const SchemaRegistry = defineState({ name: 'hexafield.conjure.SchemaRegistry', initial: { schemas: {} as Record }, register: (schema: JSONSchemaType, label?: string, description?: string) => { - const hash = contentHash(schema) + const hash = contentHashJSONSchema(schema as any /**@todo unify json schema types */) + console.log('Registered schema:', label) + if (getState(SchemaRegistry).schemas[hash]) return hash // don't overwrite existing schemas getMutableState(SchemaRegistry).schemas[hash].set({ hash, label: label || 'Untitled', @@ -28,7 +41,96 @@ export const SchemaRegistry = defineState({ forget: (hash: SHA256Hash) => { getMutableState(SchemaRegistry).schemas[hash].set(none) + if (!P2P_API.client) return + P2P_API.client.delete({ source: hash, predicate: SCHEMA_PREDICATE }).then(async () => { + console.log('deleted:', hash) + }) + }, + + /** + * Find a schema that does not fail validation for an incoming data object. + */ + findMatchingSchema: (data: any) => { + const schemaState = getState(SchemaRegistry).schemas + const matchingHash = Object.keys(schemaState).find((hash: SHA256Hash) => { + const schema = schemaState[hash] + if (!validators[hash]) { + validators[hash] = compileSchema(schema.schema) + } + try { + const { valid } = validators[hash].validate(data) + return valid + } catch (e) { + // validation failed + return null + } + }) + return matchingHash + }, + + findOrGenerateMatchingSchema: (data: any, label?: string, description?: string) => { + const existingSchema = SchemaRegistry.findMatchingSchema(data) + if (existingSchema) return getState(SchemaRegistry).schemas[existingSchema] + const newSchema = generateJsonSchema(data) + SchemaRegistry.register(newSchema, label, description) + return newSchema }, - extension: syncStateWithLocalStorage(['schemas']) + reactor: () => { + const schemaState = useMutableState(SchemaRegistry).schemas + const apiReady = useMutableState(P2P_API).ready.value + + useEffect(() => { + registerKnownSchemas() + }, []) + + useEffect(() => { + if (!apiReady) return + P2P_API.client.find({ predicate: SCHEMA_PREDICATE }).then((sources) => { + sources.forEach(async (source) => { + P2P_API.client + .get({ source, predicate: SCHEMA_PREDICATE }) + .then(async (response: object) => { + if (!response) return + const { schema, label, description } = response as any + SchemaRegistry.register(schema, label, description) + }) + .catch((e) => { + console.error('Failed to retrieve schema:', e) + }) + }) + }) + }, [apiReady]) + + if (!apiReady) return null + + return ( + <> + {schemaState.keys.map((key) => ( + + ))} + + ) + } }) + +const SyncSchema = ({ hash }: { hash: string }) => { + const schema = useMutableState(SchemaRegistry).schemas[hash].get(NO_PROXY) + + useEffect(() => { + P2P_API.client.has({ source: hash, predicate: SCHEMA_PREDICATE }).then(async (exists) => { + if (exists) return + P2P_API.client + .create({ + predicate: SCHEMA_PREDICATE, + source: hash, + target: { schema: schema.schema, label: schema.label, description: schema.description } + }) + .catch((e) => { + console.error('Failed to create schema:', e) + }) + }) + }, [schema.description, schema.label, schema.schema]) + + return null +} diff --git a/src/tools/registries/TargetRegistry.ts b/src/tools/registries/TargetRegistry.ts index 5d66dca..571c557 100644 --- a/src/tools/registries/TargetRegistry.ts +++ b/src/tools/registries/TargetRegistry.ts @@ -1,4 +1,4 @@ -import { defineState, getMutableState } from '@ir-engine/hyperflux' +import { defineState, getMutableState, getState } from '@ir-engine/hyperflux' import { JSONSchemaType } from '../json-schema/JSONSchema' import { SchemaRegistry } from '../registries/SchemaRegistry' @@ -6,7 +6,7 @@ export type TargetSchema = { hash: string label: string value: JSONSchemaType - deserialize: (data: Record>) => void + deserialize: (data: T) => void } export type TargetSchemaDefinition = Omit, 'hash'> @@ -27,5 +27,11 @@ export const TargetRegistry = defineState({ hash } as TargetSchema }) + }, + + run: (hash: string, data: T) => { + const target = getState(TargetRegistry)[hash] + if (!target) throw new Error(`Tool not found: ${hash}`) + return target.deserialize(data) } }) diff --git a/src/tools/registries/ToolRegistry.tsx b/src/tools/registries/ToolRegistry.tsx index 2490ebd..05a3f77 100644 --- a/src/tools/registries/ToolRegistry.tsx +++ b/src/tools/registries/ToolRegistry.tsx @@ -1,19 +1,15 @@ import transform from '@hexafield/jsonpath-object-transform' -import { - defineState, - getMutableState, - getState, - none, - syncStateWithLocalStorage, - useHookstate -} from '@ir-engine/hyperflux' -import { useEffect } from 'react' +import { defineState, getMutableState, getState, NO_PROXY, none, useMutableState } from '@ir-engine/hyperflux' +import React, { useEffect } from 'react' +import { P2P_API } from '../../api/CRUD' import { JSONSchemaType } from '../json-schema/JSONSchema' -import { contentHash } from '../json-schema/contentHash' +import { contentHash, contentHashJSONSchema } from '../json-schema/contentHash' import { createDynamicWebworker } from '../utils/createDynamicWebworker' import { hashFunctionSource } from '../utils/hashFunction' import { SchemaRegistry, SHA256Hash } from './SchemaRegistry' +export const TOOL_PREDICATE = 'conjure://tool' + export type Stringify = string & { __fnSignature: Signature } @@ -39,8 +35,8 @@ export const ToolRegistry = defineState({ create: async (tool: Omit): Promise => { const { label, description, input, output, transformation } = tool - const inputHash = contentHash(input) as SHA256Hash - const outputHash = contentHash(output) as SHA256Hash + const inputHash = contentHashJSONSchema(input as any /**@todo unify json schema types */) as SHA256Hash + const outputHash = contentHashJSONSchema(output as any /**@todo unify json schema types */) as SHA256Hash const transformationHash = typeof transformation === 'string' ? ((await hashFunctionSource(transformation)) as FunctionHash) @@ -63,6 +59,8 @@ export const ToolRegistry = defineState({ transformationHash } + console.log('Registered tool:', label) + getMutableState(ToolRegistry).tools[serializedTool.hash].set(serializedTool) return serializedTool.hash @@ -70,6 +68,11 @@ export const ToolRegistry = defineState({ forget: (hash: SHA256Hash) => { getMutableState(ToolRegistry).tools[hash].set(none) + + if (!P2P_API.client) return + P2P_API.client.delete({ source: hash, predicate: TOOL_PREDICATE }).then(async () => { + console.log('deleted:', hash) + }) }, run: async (hash: SHA256Hash, input: Input): Promise => { @@ -102,21 +105,92 @@ export const ToolRegistry = defineState({ } }, - extension: syncStateWithLocalStorage(['tools']), - reactor: () => { - const state = useHookstate(getMutableState(ToolRegistry).tools) + const toolState = useMutableState(ToolRegistry).tools + const apiReady = useMutableState(P2P_API).ready.value useEffect(() => { - const tools = getState(ToolRegistry).tools - for (const [hash, tool] of Object.entries(tools)) { - if (!getState(SchemaRegistry).schemas[tool.inputHash]) - SchemaRegistry.register(tool.input, tool.label, tool.description) - if (!getState(SchemaRegistry).schemas[tool.outputHash]) - SchemaRegistry.register(tool.output, tool.label, tool.description) - } - }, [state]) + if (!apiReady) return + P2P_API.client.find({ predicate: TOOL_PREDICATE }).then((sources) => { + sources.forEach(async (source) => { + P2P_API.client + .get({ source, predicate: TOOL_PREDICATE }) + .then(async (response: object) => { + if (!response) return + const { label, description, input, output, transformation } = response as Tool + ToolRegistry.create({ + label, + description, + input, + output, + transformation + }) + }) + .catch((e) => { + console.error('Failed to retrieve schema:', e) + }) + }) + }) + }, [apiReady]) + + return ( + <> + {toolState.keys.map((key) => ( + + ))} + + ) } }) +const SyncTool = ({ hash }: { hash: string }) => { + const tool = useMutableState(ToolRegistry).tools[hash].get(NO_PROXY) + const apiReady = useMutableState(P2P_API).ready.value + + useEffect(() => { + if (!getState(SchemaRegistry).schemas[tool.inputHash]) + SchemaRegistry.register(tool.input as JSONSchemaType, tool.label, tool.description) + if (!getState(SchemaRegistry).schemas[tool.outputHash]) + SchemaRegistry.register(tool.output as JSONSchemaType, tool.label, tool.description) + + if (!apiReady) return + + P2P_API.client.has({ source: hash, predicate: TOOL_PREDICATE }).then(async (exists) => { + if (exists) return + P2P_API.client + .create({ + predicate: TOOL_PREDICATE, + source: hash, + target: { + hash: tool.hash, + label: tool.label, + description: tool.description, + input: tool.input, + output: tool.output, + transformation: tool.transformation, + inputHash: tool.inputHash, + outputHash: tool.outputHash, + transformationHash: tool.transformationHash + } + }) + .catch((e) => { + console.error('Failed to create tool:', e) + }) + }) + }, [ + apiReady, + tool.inputHash, + tool.outputHash, + tool.input, + tool.output, + tool.label, + tool.description, + tool.hash, + tool.transformation, + tool.transformationHash + ]) + + return null +} + const workers = {} as Record>> diff --git a/src/tools/schemas/KnownSchemas.ts b/src/tools/schemas/KnownSchemas.ts new file mode 100644 index 0000000..738036a --- /dev/null +++ b/src/tools/schemas/KnownSchemas.ts @@ -0,0 +1,21 @@ +import { JSONSchemaType } from '../json-schema/JSONSchema' +import { SchemaRegistry } from '../registries/SchemaRegistry' +import KumuLimicon2024 from './limicon_2024.json' +import KumuLimicon2025 from './limicon_2025.json' +import MurmurationsOrganizationsV1_0_0 from './organizations_schema-v1.0.0.json' +import MurmurationsPeopleV0_1_0 from './people_schema-v0.1.0.json' + +export function registerKnownSchemas() { + SchemaRegistry.register( + MurmurationsOrganizationsV1_0_0 as JSONSchemaType, + 'Murmurations - Organizations', + MurmurationsOrganizationsV1_0_0.description + ) + SchemaRegistry.register( + MurmurationsPeopleV0_1_0 as JSONSchemaType, + 'Murmurations - People', + MurmurationsPeopleV0_1_0.description + ) + SchemaRegistry.register(KumuLimicon2024 as JSONSchemaType, 'Kumu - Limicon 2024', 'Limicon 2024') + SchemaRegistry.register(KumuLimicon2025 as JSONSchemaType, 'Kumu - Limicon 2025', 'Limicon 2025') +} diff --git a/src/tools/schemas/limicon_2024.json b/src/tools/schemas/limicon_2024.json new file mode 100644 index 0000000..ef6ea08 --- /dev/null +++ b/src/tools/schemas/limicon_2024.json @@ -0,0 +1,213 @@ +{ + "type": "object", + "properties": { + "connections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Id": { + "type": "integer" + }, + "From": { + "type": "integer" + }, + "To": { + "type": "integer" + }, + "Name From": { + "type": "string" + }, + "Name To": { + "type": "string" + }, + "Initial Date": { + "type": "string" + }, + "Last Date": { + "type": "string" + }, + "Type": { + "type": "string" + }, + "Weight": { + "type": "integer" + }, + "Connection Strength": { + "type": "string" + }, + "Anything to add about this relationship?": { + "type": "string" + }, + "Did you meet at Limicon?": { + "type": "string" + } + }, + "required": [ + "Id", + "From", + "To", + "Name From", + "Name To", + "Initial Date", + "Last Date", + "Type", + "Weight", + "Connection Strength", + "Anything to add about this relationship?", + "Did you meet at Limicon?" + ] + } + }, + "elements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Id": { + "type": "integer" + }, + "Type": { + "type": "string" + }, + "Label": { + "type": "string" + }, + "First Name": { + "type": "string" + }, + "Last Name": { + "type": "string" + }, + "Description": { + "type": "string" + }, + "Segment": { + "type": "string" + }, + "Image": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null", + "nullable": true + } + ] + }, + "Project Name": { + "type": "string" + }, + "Mailchimp Opt-In": { + "type": "boolean" + }, + "Terms and Conditions": { + "type": "boolean" + }, + "Initial Date": { + "type": "string" + }, + "Last Date": { + "type": "string" + }, + "Creation date": { + "type": "string" + }, + "What I'd like to share about myself:": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "What city, region, country are you currently living?": { + "type": "string" + }, + "How to connect:": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "A question I would like to explore at Limicon:": { + "type": "string" + }, + "Share up to 3 parts of the field that you feel yourself to be most connected with:": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "What I may offer into Limicon:": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "A few wishes I may have for Limicon:": { + "type": "string" + } + }, + "required": [ + "Id", + "Type", + "Label", + "First Name", + "Last Name", + "Description", + "Segment", + "Image", + "Project Name", + "Mailchimp Opt-In", + "Terms and Conditions", + "Initial Date", + "Last Date", + "Creation date", + "What I'd like to share about myself:", + "What city, region, country are you currently living?", + "How to connect:", + "A question I would like to explore at Limicon:", + "Share up to 3 parts of the field that you feel yourself to be most connected with:", + "What I may offer into Limicon:", + "A few wishes I may have for Limicon:" + ] + } + }, + "name": { + "type": "string" + } + }, + "required": [ + "connections", + "elements", + "name" + ] +} \ No newline at end of file diff --git a/src/tools/schemas/limicon_2025.json b/src/tools/schemas/limicon_2025.json new file mode 100644 index 0000000..c4e112b --- /dev/null +++ b/src/tools/schemas/limicon_2025.json @@ -0,0 +1,257 @@ +{ + "type": "object", + "properties": { + "connections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Id": { + "type": "integer" + }, + "From": { + "type": "integer" + }, + "To": { + "type": "integer" + }, + "Name From": { + "type": "string" + }, + "Name To": { + "type": "string" + }, + "Initial Date": { + "type": "string" + }, + "Last Date": { + "type": "string" + }, + "Type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null", + "nullable": true + } + ] + }, + "Weight": { + "type": "integer" + }, + "Connection strength": { + "type": "string" + }, + "Met through Limicon": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": [ + "Id", + "From", + "To", + "Name From", + "Name To", + "Initial Date", + "Last Date", + "Type", + "Weight", + "Connection strength", + "Met through Limicon" + ] + } + }, + "elements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Id": { + "type": "integer" + }, + "Type": { + "type": "string" + }, + "Label": { + "type": "string" + }, + "First Name": { + "type": "string" + }, + "Last Name": { + "type": "string" + }, + "Description": { + "type": "string" + }, + "Segment": { + "type": "string" + }, + "Image": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null", + "nullable": true + } + ] + }, + "Project Name": { + "type": "string" + }, + "Mailchimp Opt-In": { + "type": "boolean" + }, + "Terms and Conditions": { + "type": "boolean" + }, + "Initial Date": { + "type": "string" + }, + "Last Date": { + "type": "string" + }, + "Creation date": { + "type": "string" + }, + "1. What I'd like to share about myself": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "2. How to connect with me": { + "type": "string" + }, + "3. Where to check out my work": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "4. The city, country I am currently living in": { + "type": "string" + }, + "5. I feel most connected with these topics (choose up to 3)": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "5. Other": { + "type": "string" + }, + "6a. Needs that this network could help me with": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "6b. Elaborate on any needs you'd like": { + "type": "string" + }, + "7a. Things I can offer to this network": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "7b. Elaborate on any offers you'd like": { + "type": "string" + }, + "8. A question I would like to explore at Limicon": { + "type": "string" + }, + "9. Something I want to add to this space": { + "type": "string" + } + }, + "required": [ + "Id", + "Type", + "Label", + "First Name", + "Last Name", + "Description", + "Segment", + "Image", + "Project Name", + "Mailchimp Opt-In", + "Terms and Conditions", + "Initial Date", + "Last Date", + "Creation date", + "1. What I'd like to share about myself", + "2. How to connect with me", + "3. Where to check out my work", + "4. The city, country I am currently living in", + "5. I feel most connected with these topics (choose up to 3)", + "5. Other", + "6a. Needs that this network could help me with", + "6b. Elaborate on any needs you'd like", + "7a. Things I can offer to this network", + "7b. Elaborate on any offers you'd like", + "8. A question I would like to explore at Limicon", + "9. Something I want to add to this space" + ] + } + }, + "name": { + "type": "string" + } + }, + "required": [ + "connections", + "elements", + "name" + ] +} \ No newline at end of file diff --git a/src/tools/schemas/organizations_schema-v1.0.0.json b/src/tools/schemas/organizations_schema-v1.0.0.json new file mode 100644 index 0000000..1027e82 --- /dev/null +++ b/src/tools/schemas/organizations_schema-v1.0.0.json @@ -0,0 +1,712 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://library.murmurations.network/v2/schemas/organizations_schema-v1.0.0", + "title": "Group/Project/Organization Schema", + "description": "A schema to add regenerative economy Groups, Projects and Organizations to the Murmurations Index", + "type": "object", + "properties": { + "linked_schemas": { + "title": "Linked Schemas", + "description": "A list of schemas against which a profile must be validated (schema names must be alphanumeric with underscore(_) spacers and dash(-) semantic version separator, e.g., my_data_schema-v1.0.0)", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{7,97}-v[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "minItems": 1, + "uniqueItems": true, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "linked_schemas", "version": "1.0.0" }, + "purpose": "This field is required in all Murmurations schemas to ensure that a profile is valid and can be posted to the Index. It is the only required field in the default-v2.0.0 schema, which is the first schema used by the Index to process incoming profiles." + } + }, + "name": { + "title": "Group/Project/Organization Name", + "description": "The full name of the group, project or organization", + "type": "string", + "maxLength": 200, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "name", "version": "1.0.0" }, + "context": ["https://schema.org/name"], + "purpose": "The common name that is generally used to refer to the entity, organization, project, item, etc., which can be a living being, a legal entity, an object (real or virtual) or even a good or service." + } + }, + "nickname": { + "title": "Nickname", + "description": "The familiar name of the group, project or organization", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "nickname", "version": "1.0.0" }, + "purpose": "The familiar name that is generally used to refer to the entity, organization, project or item." + } + }, + "primary_url": { + "title": "Primary URL", + "description": "The unique and definitive website address for the group (e.g., https://my-group.org or https://some-host.net/my-org)", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "primary_url", "version": "1.0.0" }, + "context": ["https://schema.org/identifier"], + "purpose": "The primary URL is used to identify the entity or item, and is usually its main website address or, if the entity doesn't have a website it can be a web page that is well-known to be linked to the entity (e.g. a Facebook page)." + } + }, + "tags": { + "title": "Tags/Type", + "description": "Keywords that describe the group such as its type, searchable in the Murmurations index", + "type": "array", + "items": { "type": "string", "maxLength": 100 }, + "uniqueItems": true, + "minItems": 1, + "maxItems": 100, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "tags", "version": "1.0.0" }, + "context": ["https://schema.org/keywords"], + "purpose": "Tags holds a list of unique keywords that are used to describe any aspect of the entity, such that there is enough information to fit the entity into a variety of data taxonomies." + } + }, + "urls": { + "title": "Other URLs", + "description": "URLs for the group's other websites, social media, etc.", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "title": "URL Name", + "description": "The name of what this URL is for (e.g., type of website such as work, personal, etc.)", + "type": "string" + }, + "url": { + "title": "URL", + "description": "The URL (starting with http:// or https://) itself", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*" + } + }, + "required": ["url"] + }, + "uniqueItems": true, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "urls", "version": "1.0.0" }, + "context": ["https://schema.org/url"], + "purpose": "URLs can be used to link the named entity to its presence on the web. For instance a group may link to informational sites and social media related to it. An individual may link to personal and work-related websites. In the case of an item or service, URLs can provide further information about them." + } + }, + "description": { + "title": "Description", + "description": "A short description of the group", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "description", "version": "1.0.0" }, + "context": ["http://schema.org/description"], + "purpose": "The Description field can be used to provided a description of the item, entity, organization, project, etc. We have chosen not to add a maximum length but aggregators may snip the first ~160 characters of this field to provide a summary in directory listings or maps, so make sure the first sentence provides a good overview of the entity you are describing." + } + }, + "mission": { + "title": "Mission/Purpose", + "description": "A short statement of why the group exists and its goals", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "mission", "version": "1.0.0" }, + "context": ["https://en.wikipedia.org/wiki/Mission_statement"], + "purpose": ":The mission describes the purpose of the entity: what kind of product or service it provides (for profit or not), its primary customers or market, and its geographical region of operation." + } + }, + "status": { + "title": "Status", + "description": "The current status of the group", + "type": "string", + "enum": ["active", "completed", "cancelled", "on_hold", "in_planning"], + "enumNames": ["Active", "Completed", "Cancelled", "On hold", "In planning"], + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "status", "version": "1.0.0" }, + "purpose": "Status defines the current state of a project, organization, event, etc." + } + }, + "full_address": { + "title": "Full Address", + "description": "The complete address of the group in a single text field as you would write it on an envelope, including the street address, city, postal code, country, etc.", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "full_address", "version": "1.0.0" }, + "context": ["https://schema.org/address"], + "purpose": "Address captures the complete physical address of an entity. It is used to identify the location of the entity (e.g., a street address or a virtual location in a metaverse)." + } + }, + "country_iso_3166": { + "title": "Country (2 letters)", + "description": "The two-letter country code according to the ISO 3166-1 standard where the group is located", + "type": "string", + "enum": [ + "AD", + "AE", + "AF", + "AG", + "AI", + "AL", + "AM", + "AO", + "AQ", + "AR", + "AS", + "AT", + "AU", + "AW", + "AX", + "AZ", + "BA", + "BB", + "BD", + "BE", + "BF", + "BG", + "BH", + "BI", + "BJ", + "BL", + "BM", + "BN", + "BO", + "BQ", + "BR", + "BS", + "BT", + "BV", + "BW", + "BY", + "BZ", + "CA", + "CC", + "CD", + "CF", + "CG", + "CH", + "CI", + "CK", + "CL", + "CM", + "CN", + "CO", + "CR", + "CU", + "CV", + "CW", + "CX", + "CY", + "CZ", + "DE", + "DJ", + "DK", + "DM", + "DO", + "DZ", + "EC", + "EE", + "EG", + "EH", + "ER", + "ES", + "ET", + "FI", + "FJ", + "FK", + "FM", + "FO", + "FR", + "GA", + "GB", + "GD", + "GE", + "GF", + "GG", + "GH", + "GI", + "GL", + "GM", + "GN", + "GP", + "GQ", + "GR", + "GS", + "GT", + "GU", + "GW", + "GY", + "HK", + "HM", + "HN", + "HR", + "HT", + "HU", + "ID", + "IE", + "IL", + "IM", + "IN", + "IO", + "IQ", + "IR", + "IS", + "IT", + "JE", + "JM", + "JO", + "JP", + "KE", + "KG", + "KH", + "KI", + "KM", + "KN", + "KP", + "KR", + "KW", + "KY", + "KZ", + "LA", + "LB", + "LC", + "LI", + "LK", + "LR", + "LS", + "LT", + "LU", + "LV", + "LY", + "MA", + "MC", + "MD", + "ME", + "MF", + "MG", + "MH", + "MK", + "ML", + "MM", + "MN", + "MO", + "MP", + "MQ", + "MR", + "MS", + "MT", + "MU", + "MV", + "MW", + "MX", + "MY", + "MZ", + "NA", + "NC", + "NE", + "NF", + "NG", + "NI", + "NL", + "NO", + "NP", + "NR", + "NU", + "NZ", + "OM", + "PA", + "PE", + "PF", + "PG", + "PH", + "PK", + "PL", + "PM", + "PN", + "PR", + "PS", + "PT", + "PW", + "PY", + "QA", + "RE", + "RO", + "RS", + "RU", + "RW", + "SA", + "SB", + "SC", + "SD", + "SE", + "SG", + "SH", + "SI", + "SJ", + "SK", + "SL", + "SM", + "SN", + "SO", + "SR", + "SS", + "ST", + "SV", + "SX", + "SY", + "SZ", + "TC", + "TD", + "TF", + "TG", + "TH", + "TJ", + "TK", + "TL", + "TM", + "TN", + "TO", + "TR", + "TT", + "TV", + "TW", + "TZ", + "UA", + "UG", + "UM", + "US", + "UY", + "UZ", + "VA", + "VC", + "VE", + "VG", + "VI", + "VN", + "VU", + "WF", + "WS", + "YE", + "YT", + "ZA", + "ZM", + "ZW" + ], + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "country_iso_3166", "version": "1.0.0" }, + "context": ["https://en.wikipedia.org/wiki/ISO_3166-1"] + } + }, + "geolocation": { + "title": "Geolocation Coordinates", + "description": "The geo-coordinates (latitude \u0026 longitude) of the primary location of the group", + "type": "object", + "properties": { + "lat": { + "title": "Latitude", + "description": "A decimal amount between -90 and 90", + "type": "number", + "minimum": -90, + "maximum": 90, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "latitude", "version": "1.0.0" }, + "context": ["https://schema.org/latitude"] + } + }, + "lon": { + "title": "Longitude", + "description": "A decimal amount between -180 and 180", + "type": "number", + "minimum": -180, + "maximum": 180, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "longitude", "version": "1.0.0" }, + "context": ["https://schema.org/longitude"] + } + } + }, + "required": ["lat", "lon"], + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "geolocation", "version": "1.0.0" }, + "context": ["https://schema.org/latitude", "https://schema.org/longitude", "https://schema.org/GeoCoordinates"] + } + }, + "image": { + "title": "Image/Logo", + "description": "An image URL (starting with https:// or http://) for the group's logo, preferably a square", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "image", "version": "1.0.0" }, + "context": ["https://schema.org/image"], + "purpose": "An image that is generally used to refer to the entity, organization, project, item, etc." + } + }, + "header_image": { + "title": "Header Image", + "description": "The URL of a header image (normally 1500px wide and 500px high) starting with http:// or https://", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "header_image", "version": "1.0.0" }, + "context": ["https://schema.org/image"], + "purpose": "An image used in the background of a header section for an organization, project, item, etc." + } + }, + "images": { + "title": "Other images", + "description": "URLs for other images (starting with https:// or http://)", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "title": "Image Name", + "description": "Description of the image", + "type": "string", + "maxLength": 100 + }, + "url": { + "title": "URL", + "description": "A URL of the image starting with http:// or https://", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*" + } + }, + "required": ["url"] + }, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "images", "version": "1.0.0" }, + "context": ["https://schema.org/image"] + } + }, + "rss": { + "title": "RSS URL", + "description": "A URL (starting with https:// or http://) for the Really Simple Syndication feed for the group (usually found at a URL such as https://my-group.org/feed)", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "rss", "version": "1.0.0" }, + "context": ["https://en.wikipedia.org/wiki/RSS"] + } + }, + "relationships": { + "title": "Relationships", + "description": "A list of relationships between this group (the subject) and various other entities (objects)", + "type": "array", + "items": { + "type": "object", + "properties": { + "predicate_url": { + "title": "Predicate URL", + "description": "A URL defining the predicate of the relationship (e.g., https://schema.org/member or https://schema.org/knows)", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*" + }, + "object_url": { + "title": "Object URL", + "description": "The URL (ideally the Primary URL) of the object of this relationship (must start with http:// or https://, e.g., https://alice.net)", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*" + } + }, + "required": ["predicate_url", "object_url"] + }, + "uniqueItems": true, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "relationships", "version": "1.0.0" }, + "purpose": "Relationships describe the links between a Subject and an Object. In a Murmurations profile the entity publishing these relationships is the Subject. The object_url should be the Primary URL of the receiving entity (e.g., https://alice.net), and the predicate should be a URL which defines the relationship the subject has with the object (e.g. https://schema.org/knows)." + } + }, + "starts_at": { + "title": "Start Date/Time", + "description": "The date and time the group was created (as a Unix timestamp, e.g., 1651848477)", + "type": "number", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "starts_at", "version": "1.0.0" }, + "context": ["https://schema.org/startDate"], + "purpose": "A starting date and time for an entity (founding date, birth date), event, etc. To specify just the year, assume a date of 1 January at the beginning of the day (e.g., 1672531200 for Sun 01 Jan 2023 00:00:00 GMT+0000)." + } + }, + "ends_at": { + "title": "End Date/Time", + "description": "The date and time the group ceased to exist (as a Unix timestamp, e.g., 1651848477)", + "type": "number", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "ends_at", "version": "1.0.0" }, + "context": ["https://schema.org/endDate"], + "purpose": "An ending date and time for an entity (closing date, death), event, etc. To specify just the year, assume a date of 1 January at the beginning of the day (e.g., 1672531200 for Sun 01 Jan 2023 00:00:00 GMT+0000)." + } + }, + "contact_details": { + "title": "Contact Details", + "description": "The contact details for the group", + "type": "object", + "properties": { + "email": { + "title": "Email Address", + "description": "A valid email address (public)", + "type": "string" + }, + "contact_form": { + "title": "Contact Form", + "description": "A webpage (starting with https:// or http://) with a contact form", + "type": "string", + "pattern": "^https?://.*" + } + }, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "contact_details", "version": "1.0.0" }, + "purpose": "Provides a contact method for an entity." + } + }, + "telephone": { + "title": "Telephone Number", + "description": "A landline or mobile phone number, specified in international dialing format (e.g., +1 212 555 1212)", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "telephone", "version": "1.0.0" }, + "context": ["https://schema.org/telephone"], + "purpose": "A phone number that can be used to contact a person or organization. The number should be provided in international dialing format (e.g., the US telephone number (212) 555-1212 should be formatted as +1 212 555 1212)." + } + }, + "geographic_scope": { + "title": "Geographic Scope", + "description": "The geographic scope of the group is defined by the sphere of the group's activities", + "type": "string", + "enum": ["local", "regional", "national", "international"], + "enumNames": ["Local", "Regional", "National", "International"], + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "geographic_scope", "version": "1.0.0" }, + "purpose": "An entity will generally have a scope in which it operates from a local up to a global range. An item will generally be available within a specific scope as well. For example, perishable food will be available in a local or possibly regional market, but not in a national or international market." + } + }, + "unique_id": { + "title": "Unique ID", + "description": "The unique identifier of the entity, optionally defined by the group recording/tracking the entity", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "unique_id", "version": "1.0.0" }, + "context": ["https://schema.org/identifier"], + "purpose": "The unique ID is an identifier applied to an entity by a group that wishes to keep track of the entity. For example, an association of businesses will apply a unique ID to each member business in order to identify them for internal purposes." + } + } + }, + "required": ["linked_schemas", "name"], + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network/" + }, + "schema": { + "name": "organizations_schema-v1.0.0", + "purpose": "To map Groups, Projects and Organizations within the regenerative economy", + "url": "https://murmurations.network" + } + } +} diff --git a/src/tools/schemas/people_schema-v0.1.0.json b/src/tools/schemas/people_schema-v0.1.0.json new file mode 100644 index 0000000..b86e266 --- /dev/null +++ b/src/tools/schemas/people_schema-v0.1.0.json @@ -0,0 +1,669 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://library.murmurations.network/v2/schemas/people_schema-v0.1.0", + "title": "People Schema", + "description": "A schema to add individuals in the regenerative economy to the Murmurations Index", + "type": "object", + "properties": { + "linked_schemas": { + "title": "Linked Schemas", + "description": "A list of schemas against which a profile must be validated (schema names must be alphanumeric with underscore(_) spacers and dash(-) semantic version separator, e.g., my_data_schema-v1.0.0)", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{7,97}-v[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "minItems": 1, + "uniqueItems": true, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "linked_schemas", "version": "1.0.0" }, + "purpose": "This field is required in all Murmurations schemas to ensure that a profile is valid and can be posted to the Index. It is the only required field in the default-v2.0.0 schema, which is the first schema used by the Index to process incoming profiles." + } + }, + "name": { + "title": "Full Name", + "description": "The full name of the person", + "type": "string", + "maxLength": 200, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "name", "version": "1.0.0" }, + "context": ["https://schema.org/name"], + "purpose": "The common name that is generally used to refer to the entity, organization, project, item, etc., which can be a living being, a legal entity, an object (real or virtual) or even a good or service." + } + }, + "nickname": { + "title": "Nickname", + "description": "The familiar name of the person", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "nickname", "version": "1.0.0" }, + "purpose": "The familiar name that is generally used to refer to the entity, organization, project or item." + } + }, + "primary_url": { + "title": "Primary URL", + "description": "The unique and definitive website address for the person (e.g., alice.net or some-host.net/alice)", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "primary_url", "version": "1.0.0" }, + "context": ["https://schema.org/identifier"], + "purpose": "The primary URL is used to identify the entity or item, and is usually its main website address or, if the entity doesn't have a website it can be a web page that is well-known to be linked to the entity (e.g. a Facebook page)." + } + }, + "tags": { + "title": "Tags / Skills", + "description": "Keywords that describe the person, searchable in the Murmurations index", + "type": "array", + "items": { "type": "string", "maxLength": 100 }, + "uniqueItems": true, + "minItems": 1, + "maxItems": 100, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "tags", "version": "1.0.0" }, + "context": ["https://schema.org/keywords"], + "purpose": "Tags holds a list of unique keywords that are used to describe any aspect of the entity, such that there is enough information to fit the entity into a variety of data taxonomies." + } + }, + "description": { + "title": "Description/Bio", + "description": "A short description or biography of the person", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "description", "version": "1.0.0" }, + "context": ["http://schema.org/description"], + "purpose": "The Description field can be used to provided a description of the item, entity, organization, project, etc. We have chosen not to add a maximum length but aggregators may snip the first ~160 characters of this field to provide a summary in directory listings or maps, so make sure the first sentence provides a good overview of the entity you are describing." + } + }, + "image": { + "title": "Photo/Avatar", + "description": "An image URL (starting with https:// or http://), preferably a square", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "image", "version": "1.0.0" }, + "context": ["https://schema.org/image"], + "purpose": "An image that is generally used to refer to the entity, organization, project, item, etc." + } + }, + "images": { + "title": "Other Images", + "description": "Other images (starting with https:// or http://) for this person", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "title": "Image Name", + "description": "Description of the image", + "type": "string", + "maxLength": 100 + }, + "url": { + "title": "URL", + "description": "A URL of the image starting with http:// or https://", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*" + } + }, + "required": ["url"] + }, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "images", "version": "1.0.0" }, + "context": ["https://schema.org/image"] + } + }, + "urls": { + "title": "Website Addresses/URLs", + "description": "URLs for related website(s), RSS feeds, social media, etc.", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "title": "URL Name", + "description": "The name of what this URL is for (e.g., type of website such as work, personal, etc.)", + "type": "string" + }, + "url": { + "title": "URL", + "description": "The URL (starting with http:// or https://) itself", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*" + } + }, + "required": ["url"] + }, + "uniqueItems": true, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "urls", "version": "1.0.0" }, + "context": ["https://schema.org/url"], + "purpose": "URLs can be used to link the named entity to its presence on the web. For instance a group may link to informational sites and social media related to it. An individual may link to personal and work-related websites. In the case of an item or service, URLs can provide further information about them." + } + }, + "knows_language": { + "title": "Languages Spoken", + "description": "The languages a person can read, write and speak", + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "uniqueItems": true, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "knows_language", "version": "1.0.0" }, + "context": ["https://schema.org/knowsLanguage"], + "purpose": "A list of languages spoken by a person or used in communication within and by an organization, group, etc." + } + }, + "contact_details": { + "title": "Contact Details", + "description": "The person's contact details (fill in at least one)", + "type": "object", + "properties": { + "email": { + "title": "Email Address", + "description": "A valid email address (public)", + "type": "string" + }, + "contact_form": { + "title": "Contact Form", + "description": "A webpage (starting with https:// or http://) with a contact form", + "type": "string", + "pattern": "^https?://.*" + } + }, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "contact_details", "version": "1.0.0" }, + "purpose": "Provides a contact method for an entity." + } + }, + "telephone": { + "title": "Telephone Number", + "description": "A landline or mobile phone number, specified in international dialing format (e.g., +1 212 555 1212)", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "telephone", "version": "1.0.0" }, + "context": ["https://schema.org/telephone"], + "purpose": "A phone number that can be used to contact a person or organization. The number should be provided in international dialing format (e.g., the US telephone number (212) 555-1212 should be formatted as +1 212 555 1212)." + } + }, + "street_address": { + "title": "Street Address", + "description": "The street address of the entity in a single text field as you would write it on an envelope", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "street_address", "version": "1.0.0" }, + "context": ["https://schema.org/street_address"], + "purpose": "Street address captures the physical address of an entity, without the town/city, postal code, country, etc. This is useful for mapping and other applications where the full address is not required." + } + }, + "locality": { + "title": "Locality", + "description": "The locality (city, town, village, etc.) where the entity is located", + "type": "string", + "maxLength": 100, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "locality", "version": "1.0.0" }, + "context": ["https://schema.org/addressLocality"] + } + }, + "region": { + "title": "Region", + "description": "The region (state, county, province, etc.) where the entity is located", + "type": "string", + "maxLength": 100, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "region", "version": "1.0.0" }, + "context": ["https://schema.org/addressRegion"] + } + }, + "postal_code": { + "title": "Postal Code", + "description": "The postal code for the entity's address", + "type": "string", + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "postal_code", "version": "1.0.0" }, + "context": ["https://schema.org/postalCode"], + "purpose": "Postal code captures the code used by the local postal system of the entity. This is useful for mapping and other applications where the full address is not required." + } + }, + "country_name": { + "title": "Country name", + "description": "The name of country where the entity is based", + "type": "string", + "maxLength": 100, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "country_name", "version": "1.0.0" }, + "context": ["https://schema.org/Country"], + "purpose": "A free form field to enter a country's name. The Index will try to match that text to a country's name and will store the country's two-letter ISO-3166-1 code in the Index database to enable searching by country for the entity. The name-to-ISO mapping is here: https://github.com/MurmurationsNetwork/MurmurationsServices/blob/main/services/library/static/countries.json" + } + }, + "country_iso_3166": { + "title": "Country (2 letters)", + "description": "The two-letter country code according to the ISO 3166-1 standard where the entity is located", + "type": "string", + "enum": [ + "AD", + "AE", + "AF", + "AG", + "AI", + "AL", + "AM", + "AO", + "AQ", + "AR", + "AS", + "AT", + "AU", + "AW", + "AX", + "AZ", + "BA", + "BB", + "BD", + "BE", + "BF", + "BG", + "BH", + "BI", + "BJ", + "BL", + "BM", + "BN", + "BO", + "BQ", + "BR", + "BS", + "BT", + "BV", + "BW", + "BY", + "BZ", + "CA", + "CC", + "CD", + "CF", + "CG", + "CH", + "CI", + "CK", + "CL", + "CM", + "CN", + "CO", + "CR", + "CU", + "CV", + "CW", + "CX", + "CY", + "CZ", + "DE", + "DJ", + "DK", + "DM", + "DO", + "DZ", + "EC", + "EE", + "EG", + "EH", + "ER", + "ES", + "ET", + "FI", + "FJ", + "FK", + "FM", + "FO", + "FR", + "GA", + "GB", + "GD", + "GE", + "GF", + "GG", + "GH", + "GI", + "GL", + "GM", + "GN", + "GP", + "GQ", + "GR", + "GS", + "GT", + "GU", + "GW", + "GY", + "HK", + "HM", + "HN", + "HR", + "HT", + "HU", + "ID", + "IE", + "IL", + "IM", + "IN", + "IO", + "IQ", + "IR", + "IS", + "IT", + "JE", + "JM", + "JO", + "JP", + "KE", + "KG", + "KH", + "KI", + "KM", + "KN", + "KP", + "KR", + "KW", + "KY", + "KZ", + "LA", + "LB", + "LC", + "LI", + "LK", + "LR", + "LS", + "LT", + "LU", + "LV", + "LY", + "MA", + "MC", + "MD", + "ME", + "MF", + "MG", + "MH", + "MK", + "ML", + "MM", + "MN", + "MO", + "MP", + "MQ", + "MR", + "MS", + "MT", + "MU", + "MV", + "MW", + "MX", + "MY", + "MZ", + "NA", + "NC", + "NE", + "NF", + "NG", + "NI", + "NL", + "NO", + "NP", + "NR", + "NU", + "NZ", + "OM", + "PA", + "PE", + "PF", + "PG", + "PH", + "PK", + "PL", + "PM", + "PN", + "PR", + "PS", + "PT", + "PW", + "PY", + "QA", + "RE", + "RO", + "RS", + "RU", + "RW", + "SA", + "SB", + "SC", + "SD", + "SE", + "SG", + "SH", + "SI", + "SJ", + "SK", + "SL", + "SM", + "SN", + "SO", + "SR", + "SS", + "ST", + "SV", + "SX", + "SY", + "SZ", + "TC", + "TD", + "TF", + "TG", + "TH", + "TJ", + "TK", + "TL", + "TM", + "TN", + "TO", + "TR", + "TT", + "TV", + "TW", + "TZ", + "UA", + "UG", + "UM", + "US", + "UY", + "UZ", + "VA", + "VC", + "VE", + "VG", + "VI", + "VN", + "VU", + "WF", + "WS", + "YE", + "YT", + "ZA", + "ZM", + "ZW" + ], + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "country_iso_3166", "version": "1.0.0" }, + "context": ["https://en.wikipedia.org/wiki/ISO_3166-1"] + } + }, + "geolocation": { + "title": "Geolocation Coordinates", + "description": "The geo-coordinates (latitude \u0026 longitude) of the primary location of the person", + "type": "object", + "properties": { + "lat": { + "title": "Latitude", + "description": "A decimal amount between -90 and 90", + "type": "number", + "minimum": -90, + "maximum": 90, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "latitude", "version": "1.0.0" }, + "context": ["https://schema.org/latitude"] + } + }, + "lon": { + "title": "Longitude", + "description": "A decimal amount between -180 and 180", + "type": "number", + "minimum": -180, + "maximum": 180, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "longitude", "version": "1.0.0" }, + "context": ["https://schema.org/longitude"] + } + } + }, + "required": ["lat", "lon"], + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "geolocation", "version": "1.0.0" }, + "context": ["https://schema.org/latitude", "https://schema.org/longitude", "https://schema.org/GeoCoordinates"] + } + }, + "relationships": { + "title": "Relationships", + "description": "A list of relationships between this person (subject) and various other entities (objects)", + "type": "array", + "items": { + "type": "object", + "properties": { + "predicate_url": { + "title": "Predicate URL", + "description": "A URL defining the predicate of the relationship (e.g., https://schema.org/member or https://schema.org/knows)", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*" + }, + "object_url": { + "title": "Object URL", + "description": "The URL (ideally the Primary URL) of the object of this relationship (must start with http:// or https://, e.g., https://alice.net)", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://.*" + } + }, + "required": ["predicate_url", "object_url"] + }, + "uniqueItems": true, + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network" + }, + "field": { "name": "relationships", "version": "1.0.0" }, + "purpose": "Relationships describe the links between a Subject and an Object. In a Murmurations profile the entity publishing these relationships is the Subject. The object_url should be the Primary URL of the receiving entity (e.g., https://alice.net), and the predicate should be a URL which defines the relationship the subject has with the object (e.g. https://schema.org/knows)." + } + } + }, + "required": ["linked_schemas", "name", "primary_url", "tags"], + "metadata": { + "creator": { + "name": "Murmurations Network", + "url": "https://murmurations.network/" + }, + "schema": { + "name": "people_schema-v0.1.0", + "purpose": "To map people within the regenerative economy", + "url": "https://murmurations.network" + } + } +} diff --git a/src/tools/views/PipelineView.tsx b/src/tools/views/PipelineView.tsx index 1b9ae5b..025d275 100644 --- a/src/tools/views/PipelineView.tsx +++ b/src/tools/views/PipelineView.tsx @@ -1,10 +1,14 @@ import { getMutableState, getState, hookstate, NO_PROXY, useHookstate, useMutableState } from '@ir-engine/hyperflux' import React, { useEffect } from 'react' +import PipelineCard from '../components/PipelineCard' import Tabs from '../components/Tabs' import { ToolCard } from '../components/ToolCard' -import { contentHash } from '../json-schema/contentHash' +import { contentHash, contentHashJSONSchema } from '../json-schema/contentHash' import { generateJsonSchema } from '../json-schema/generateJsonSchema' import { JSONSchemaType } from '../json-schema/JSONSchema' +import type { PipelineSpec, PipelineStage } from '../pipeline/model' +import { runPipelineSpec } from '../pipeline/runPipelineSpec' +import { PipelineRegistry } from '../registries/PipelineRegistry' import { TargetRegistry } from '../registries/TargetRegistry' import { Tool, ToolRegistry } from '../registries/ToolRegistry' @@ -38,10 +42,19 @@ interface ShareLinkProps { graphType: string isVisible: boolean createShareConfig: () => SharedConfig + selectedTool?: Tool | null + outputHash?: string | null } // Share Link Component -function ShareLinkComponent({ inputs, graphType, isVisible, createShareConfig }: ShareLinkProps): JSX.Element | null { +function ShareLinkComponent({ + inputs, + graphType, + isVisible, + createShareConfig, + selectedTool, + outputHash +}: ShareLinkProps): JSX.Element | null { const shareMessage = useHookstate(null) // Generate share file and download it @@ -81,6 +94,47 @@ function ShareLinkComponent({ inputs, graphType, isVisible, createShareConfig }: > � Download Config File + {shareMessage.get() && (
    {shareMessage.get()} @@ -96,6 +150,7 @@ function ShareLinkComponent({ inputs, graphType, isVisible, createShareConfig }: function PipelineUseView(): JSX.Element { const tools = useHookstate(getMutableState(ToolRegistry).tools) + const selectedTool = useHookstate(null) // State for multiple input sources const inputs = useHookstate([ { url: '', data: null, loading: false, errorMessage: null, schema: null, hash: null } @@ -201,7 +256,7 @@ function PipelineUseView(): JSX.Element { if (!resp.ok) throw new Error(`HTTP ${resp.status}`) const data = await resp.json() const schema = generateJsonSchema(data) - const hash = contentHash(schema) + const hash = contentHashJSONSchema(schema as any /**@todo unify json schema types */) inputs[idx].merge({ data, schema, hash, loading: false, errorMessage: null }) } catch (e: any) { inputs[idx].merge({ loading: false, errorMessage: e.message || 'Fetch failed' }) @@ -282,7 +337,7 @@ function PipelineUseView(): JSX.Element { .get({ noproxy: true }) .map((input) => input.hash) .filter((h): h is string => !!h) - const matchingTools = Object.values(tools.value as Record).filter( + const matchingTools = Object.values(tools.get(NO_PROXY) as Record).filter( (tool) => inputHashes.includes(tool.inputHash) && outputHash && tool.outputHash === outputHash ) as Tool[] @@ -294,35 +349,30 @@ function PipelineUseView(): JSX.Element { // New: Run transformation tool and create graph const runToolAndCreateGraph = async () => { - // For each input, find the matching tool and run it - const results = await Promise.all( - inputs.get(NO_PROXY).map((input) => { - if (!input.hash || !input.data) return null // Skip if no hash or data - const tool = matchingTools.find((t) => t.inputHash === input.hash && t.outputHash === outputHash) - if (!tool) { - console.error(`No matching tool found for input hash ${input.hash}`) - return null - } - try { - return ToolRegistry.run(tool.hash, input.data) - } catch (e) { - // Optionally handle error - console.error('Tool run failed', e) - } - }) - ) - // Now, create the graph using the output schema's deserialize logic - if (targetGraph) { - const dataObj: Record = {} - inputs.forEach((input, i) => { - const url = input.url.get() - dataObj[url] = results[i] + const selTool = selectedTool.get(NO_PROXY) + if (!selTool) return + // Build a pipeline spec dynamically from current inputs -> selected tool -> output per input + const stages: PipelineStage[] = [] + inputs.get(NO_PROXY).forEach((input) => { + if (!input.hash || !input.data) return + const iIdx = stages.length + stages.push({ + type: 'input', + params: { url: input.url, schemaHash: input.hash ?? undefined, data: input.data }, + next: [] }) - try { - targetGraph.deserialize(dataObj) - } catch (e) { - console.error('Graph creation failed', e) - } + const tIdx = stages.length + stages.push({ type: 'tool', toolHash: selTool.hash, params: {}, next: [] }) + const oIdx = stages.length + stages.push({ type: 'output', params: { outputHash: outputHash || undefined }, next: [] }) + stages[iIdx].next.push(tIdx) + stages[tIdx].next.push(oIdx) + }) + const spec: PipelineSpec = { stages } + try { + await runPipelineSpec(spec) + } catch (e) { + console.error('Pipeline run failed', e) } } @@ -429,7 +479,14 @@ function PipelineUseView(): JSX.Element { ) : (
      {matchingTools.map((tool) => ( - runToolAndCreateGraph()} /> + { + selectedTool.set(tool) + runToolAndCreateGraph() + }} + /> ))}
    )} @@ -439,6 +496,8 @@ function PipelineUseView(): JSX.Element { graphType={visualizationType.get()} isVisible={!!allInputsHaveTool} createShareConfig={createShareConfig} + selectedTool={(selectedTool.get(NO_PROXY) || null) as any} + outputHash={outputHash} /> ) @@ -447,16 +506,36 @@ function PipelineUseView(): JSX.Element { function PipelineLibraryView(): JSX.Element { const tools = useHookstate(getMutableState(ToolRegistry).tools) const toolList = Object.values(tools.value) as Tool[] + const editorTools = toolList.map((t) => ({ hash: t.hash, label: t.label })) + const pipelines = useMutableState(PipelineRegistry).pipelines.value + const pipelineList = Object.values(pipelines) + + const saveGraphForPipeline = (spec: PipelineSpec, pipeline: any) => { + // Register a new version for now; could update in place if desired + const newHash = PipelineRegistry.register({ + label: pipeline.label, + description: pipeline.description, + graph: spec + }) + console.log('pipeline saved:', newHash) + } return (
    -

    Available Tools

    - {toolList.length === 0 ? ( -
    No tools found.
    +

    Pipelines

    + {pipelineList.length === 0 ? ( +
    No pipelines saved.
    ) : ( -
      - {toolList.map((tool) => ( - +
        + {pipelineList.map((p) => ( +
      • + {}} + onSaveGraph={saveGraphForPipeline} + /> +
        • ))}
      )} diff --git a/src/tools/views/ToolView.tsx b/src/tools/views/ToolView.tsx index 4f43076..aa1affc 100644 --- a/src/tools/views/ToolView.tsx +++ b/src/tools/views/ToolView.tsx @@ -2,7 +2,7 @@ import { hookstate, useHookstate } from '@hookstate/core' import { useEffect } from 'react' import transform from '@hexafield/jsonpath-object-transform' -import { getState, NO_PROXY } from '@ir-engine/hyperflux' +import { getMutableState, getState, NO_PROXY } from '@ir-engine/hyperflux' import { Button } from '@ir-engine/ui' import React from 'react' import { DataTransformSection } from '../components/DataTransformSection' @@ -425,13 +425,18 @@ function ToolCreateView(): JSX.Element { } function ToolRegistryView(): JSX.Element { - const tools = useHookstate(getState(ToolRegistry).tools) + const tools = useHookstate(getMutableState(ToolRegistry).tools) + + const handleForgetSchema = (hash: string) => { + ToolRegistry.forget(hash) + } + return (

      Tool Library

        {Object.values(tools.get(NO_PROXY)).map((tool: Tool) => ( - + ))}