Introduce Data Storage API, Pipelines and Drag & Drop

.gitignore

@@ -113,4 +113,6 @@ index.ts.bak
 
 public/projects/*
 
-public/data/*
+public/data/*
+
+storage/

fileserver.ts

@@ -0,0 +1,184 @@
+import express, { Request } from 'express'
+import fs from 'fs'
+import https from 'https'
+import path from 'path'
+
+type QueryParams = {
+  source: string
+  predicate: string
+}
+
+type FileParams = QueryParams & {
+  target: string
+}
+
+// Basic config
+const PORT = 8000
+// Resolve repo-level dev certs; this file lives in packages/projects/projects/hexafield/conjure/
+const CERT_PATH = path.resolve(__dirname, '../../../../../certs/cert.pem')
+const KEY_PATH = path.resolve(__dirname, '../../../../../certs/key.pem')
+
+// Local storage directory for uploaded files
+const STORAGE_DIR = path.resolve(__dirname, 'storage')
+fs.mkdirSync(STORAGE_DIR, { recursive: true })
+
+// Minimal CORS for local dev
+const corsMiddleware: express.RequestHandler = (req, res, next) => {
+  console.log(`[fileserver] ${req.method} ${req.url}`)
+  res.header('Access-Control-Allow-Origin', req.headers.origin || '*')
+  res.header('Access-Control-Allow-Credentials', 'true')
+  res.header('Access-Control-Allow-Methods', 'GET,POST,OPTIONS')
+  res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization')
+  if (req.method === 'OPTIONS') return res.sendStatus(204)
+  next()
+}
+
+const app = express()
+app.use(corsMiddleware)
+app.use(express.json({ limit: '10mb' })) // JSON only for metadata routes
+
+const keyFor = ({ source, predicate }: QueryParams) => `${encodeURIComponent(predicate)}__${encodeURIComponent(source)}`
+
+const pathFor = (q: QueryParams) => path.join(STORAGE_DIR, `${keyFor(q)}.json`)
+
+const readFile = (q: QueryParams): FileParams | undefined => {
+  const path = pathFor(q)
+  if (!fs.existsSync(path)) return undefined
+  try {
+    return JSON.parse(fs.readFileSync(path, 'utf-8')) as FileParams
+  } catch {
+    return undefined
+  }
+}
+
+const writeFile = (q: QueryParams, data: string) => {
+  fs.writeFileSync(pathFor(q), data)
+}
+
+app.get('/health', (_req, res) => res.status(200).send('ok'))
+
+app.post('/create', (req: Request, res) => {
+  try {
+    const { source, predicate, target } = req.body as unknown as FileParams
+    if (!source || !predicate || !target) {
+      return res.status(400).json({ error: 'Missing required fields' })
+    }
+
+    const q: QueryParams = { source, predicate }
+    const finalPath = pathFor(q)
+
+    if (finalPath && fs.existsSync(finalPath)) fs.unlinkSync(finalPath)
+
+    writeFile(q, target)
+
+    return res.status(201).json({ ok: true })
+  } catch (e) {
+    console.error('CREATE error', e)
+    return res.status(500).json({ error: 'Internal error' })
+  }
+})
+
+// Get (download)
+app.post('/get', (req, res) => {
+  try {
+    const { source, predicate } = req.body as QueryParams
+    if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' })
+
+    const path = pathFor({ source, predicate })
+    if (!path || !fs.existsSync(path)) return res.status(404).json({ error: 'Not found' })
+
+    const file = readFile({ source, predicate })
+    if (!file) return res.status(404).json({ error: 'Not found' })
+
+    return res.status(200).json(file)
+  } catch (e) {
+    console.error('GET error', e)
+    return res.status(500).json({ error: 'Internal error' })
+  }
+})
+
+// Find (return results for matching predicate)
+app.post('/find', (req, res) => {
+  try {
+    const { predicate } = req.body as { predicate?: string }
+    if (!predicate) return res.status(400).json({ error: 'Missing required fields' })
+
+    // Scan storage dir for all .json meta files
+    const results: Array<string> = []
+    const files = fs.readdirSync(STORAGE_DIR)
+    for (const file of files) {
+      if (!file.endsWith('.json')) continue
+      const encodedPredicate = encodeURIComponent(predicate)
+      if (file.startsWith(encodedPredicate)) {
+        results.push(decodeURIComponent(file.slice(encodedPredicate.length + 2, -5)))
+      }
+    }
+    return res.status(200).json({ ok: true, results })
+  } catch (e) {
+    console.error('FIND error', e)
+    return res.status(500).json({ error: 'Internal error' })
+  }
+})
+
+app.post('/has', (req, res) => {
+  try {
+    const { source, predicate } = req.body as QueryParams
+    if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' })
+
+    const file = fs.existsSync(pathFor({ source, predicate }))
+    if (!file) return res.status(200).json({ ok: false })
+
+    return res.status(200).json({ ok: true })
+  } catch (e) {
+    console.error('HAS error', e)
+    return res.status(500).json({ error: 'Internal error' })
+  }
+})
+
+// Replace
+app.post('/replace', (req: Request, res) => {
+  try {
+    const { source, predicate, target } = req.body as unknown as FileParams
+    if (!source || !predicate || !target) {
+      return res.status(400).json({ error: 'Missing required fields' })
+    }
+    const q: QueryParams = { source, predicate }
+    const path = pathFor(q)
+    if (!path) {
+      return res.status(404).json({ error: 'Not found' })
+    }
+    // Remove old file
+    if (path && fs.existsSync(path)) fs.unlinkSync(path)
+
+    writeFile(q, target)
+
+    return res.status(200).json({ ok: true })
+  } catch (e) {
+    console.error('REPLACE error', e)
+    return res.status(500).json({ error: 'Internal error' })
+  }
+})
+
+// Delete
+app.post('/delete', (req, res) => {
+  try {
+    const { source, predicate } = req.body as QueryParams
+    if (!source || !predicate) return res.status(400).json({ error: 'Missing required fields' })
+    const q: QueryParams = { source, predicate }
+
+    const path = pathFor(q)
+    if (fs.existsSync(path)) fs.unlinkSync(path)
+
+    return res.status(200).json({ ok: true })
+  } catch (e) {
+    console.error('DELETE error', e)
+    return res.status(500).json({ error: 'Internal error' })
+  }
+})
+
+// HTTPS server
+const key = fs.readFileSync(KEY_PATH)
+const cert = fs.readFileSync(CERT_PATH)
+https.createServer({ key, cert }, app).listen(PORT, () => {
+  console.log(`[fileserver] HTTPS listening on https://localhost:${PORT}`)
+})

package.json

@@ -12,6 +12,7 @@
     "build:worker": "esbuild src/tools/graph/forcegraph/ForceGraphWorker.ts --bundle --format=esm --outfile=src/tools/graph/forcegraph/ForceGraphWorker.bundle.js --loader:.wasm=binary",
     "build:dag-worker": "esbuild src/tools/graph/dag/DagWorker.ts --bundle --format=esm --outfile=src/tools/graph/dag/DagWorker.bundle.js --loader:.wasm=binary",
     "start": "tsx server.js",
+    "fileserver": "tsx fileserver.ts",
     "format": "prettier --write \"**/*.{ts,tsx}\"",
     "format-scss": "stylelint \"**/*.scss\" --fix",
     "format-staged": "lint-staged",
@@ -24,13 +25,18 @@
     "@coasys/ad4m-connect": "^0.10.1-release-candidate-4",
     "@hexafield/jsonpath-object-transform": "^2.0.0",
     "@mlc-ai/web-llm": "^0.2.79",
+    "canonicalize": "^2.1.0",
     "d3": "7.9.0",
     "d3-force-3d": "^3.0.5",
     "dat.gui": "^0.7.9",
+    "express": "^4.19.2",
+    "json-schema-library": "^10.2.1",
     "json-schema-to-ts": "^3.1.1",
     "jsonpath-plus": "10.3.0",
     "maxrects-packer": "2.7.3",
     "monaco-editor": "^0.52.2",
+    "react-rnd": "^10.5.2",
+    "reactflow": "^11.10.0",
     "ts-morph": "^26.0.0"
   },
   "license": "ISC",

src/ad4m/useADAM.tsx

@@ -1,7 +1,36 @@
-import { Ad4mClient, Agent } from '@coasys/ad4m'
+import { Ad4mClient, Agent, Link, LinkMutations, Literal } from '@coasys/ad4m'
 import Ad4mConnectUI, { Ad4mConnectElement, getAd4mClient } from '@coasys/ad4m-connect'
-import { defineState, getMutableState, useHookstate, useMutableState } from '@ir-engine/hyperflux'
+import { defineState, getMutableState, getState, useHookstate, useMutableState } from '@ir-engine/hyperflux'
 import { useEffect } from 'react'
+import { CRUD_API, P2P_API } from '../api/CRUD'
+
+export const blobToDataURL = (blob: Blob): Promise<string> => {
+  return new Promise<string>((resolve, reject) => {
+    const reader = new FileReader()
+    reader.onload = () => {
+      const result = reader.result as string
+      const base64 = result.split(',')[1]
+      resolve(base64 as string)
+    }
+    reader.onerror = () => reject(reader.error)
+    reader.onabort = () => reject(new Error('Read aborted'))
+    reader.readAsDataURL(blob)
+  })
+}
+
+export const dataURLToBlob = (dataURL: string): Promise<Blob> => {
+  return new Promise<Blob>((resolve, reject) => {
+    const arr = dataURL.split(',')
+    const mime = arr[0].match(/:(.*?);/)?.[1]
+    const bstr = atob(arr[1])
+    let n = bstr.length
+    const u8arr = new Uint8Array(n)
+    while (n--) {
+      u8arr[n] = bstr.charCodeAt(n)
+    }
+    resolve(new Blob([u8arr], { type: mime }))
+  })
+}
 
 export const AdamClientState = defineState({
   name: 'hexafield.adam-template.AdamClientState',
@@ -69,7 +98,123 @@ export const AgentState = defineState({
 
       adam.agent.me().then((response) => {
         getMutableState(AgentState).set(response)
+        P2P_API.client = AgentBlobAPI
+        getMutableState(P2P_API).ready.set(true)
+        console.log('ADAM storage method ready')
       })
     }, [adam])
   }
 })
+
+export const AgentBlobAPI: CRUD_API = {
+  create: async (args) => {
+    const client = getState(AdamClientState)
+    if (!client) throw new Error('AD4M client not initialized')
+
+    const newLink = new Link({
+      source: args.source,
+      predicate: args.predicate,
+      target: Literal.from(args.target).toUrl()
+    })
+
+    await client.agent.mutatePublicPerspective({
+      additions: [newLink],
+      removals: []
+    } as LinkMutations)
+  },
+
+  has: async (args) => {
+    const client = getState(AdamClientState)
+    if (!client) throw new Error('AD4M client not initialized')
+
+    const myPerspectives = await client.agent.me()
+    if (!myPerspectives) throw new Error('Agent not initialized')
+
+    const myLinks = myPerspectives.perspective!.links
+
+    const link = myLinks.find((link) => link.data.source === args.source && link.data.predicate === args.predicate)
+    return !!link
+  },
+
+  get: async (args) => {
+    const client = getState(AdamClientState)
+    if (!client) throw new Error('AD4M client not initialized')
+
+    const myPerspectives = await client.agent.me()
+    if (!myPerspectives) throw new Error('Agent not initialized')
+
+    const myLinks = myPerspectives.perspective!.links
+
+    const link = myLinks.find((link) => link.data.source === args.source && link.data.predicate === args.predicate)
+    if (!link) return undefined // not found
+
+    return Literal.fromUrl(link.data.target).get()
+  },
+
+  find: async (args) => {
+    const client = getState(AdamClientState)
+    if (!client) throw new Error('AD4M client not initialized')
+
+    const myPerspectives = await client.agent.me()
+    if (!myPerspectives) throw new Error('Agent not initialized')
+
+    const myLinks = myPerspectives.perspective!.links
+
+    const foundLinks = myLinks.filter((link) => link.data.predicate === args.predicate)
+    return foundLinks.map((link) => link.data.source)
+  },
+
+  replace: async (args) => {
+    const client = getState(AdamClientState)
+    if (!client) throw new Error('AD4M client not initialized')
+
+    const myPerspectives = await client.agent.me()
+    if (!myPerspectives) throw new Error('Agent not initialized')
+
+    const myLinks = myPerspectives.perspective!.links
+
+    const currentLink = myLinks.find(
+      (link) => link.data.source === args.source && link.data.predicate === args.predicate
+    )
+    if (!currentLink) throw new Error('Current link not found')
+
+    const removals = [currentLink]
+
+    const newLink = new Link({
+      source: args.source,
+      predicate: args.predicate,
+      target: Literal.from(args.target).toUrl()
+    })
+
+    await client.agent.mutatePublicPerspective({
+      additions: [newLink],
+      removals: removals
+    })
+  },
+
+  delete: async (args) => {
+    const client = getState(AdamClientState)
+    if (!client) throw new Error('AD4M client not initialized')
+
+    const myPerspectives = await client.agent.me()
+    if (!myPerspectives) throw new Error('Agent not initialized')
+
+    const myLinks = myPerspectives.perspective!.links
+
+    const currentLink = myLinks.find(
+      (link) => link.data.source === args.source && link.data.predicate === args.predicate
+    )
+    if (!currentLink) throw new Error('Current link not found')
+
+    await client.agent.mutatePublicPerspective({
+      additions: [],
+      removals: [currentLink]
+    })
+  }
+}
+
+globalThis.AdamClientState = AdamClientState
+globalThis.AgentState = AgentState
+globalThis.AgentBlobAPI = AgentBlobAPI
+
+// wipe data: `AgentBlobAPI.find({ predicate: 'conjure://schema' }).then(sources => sources.forEach(source => AgentBlobAPI.delete({ source, predicate: 'conjure://schema' })))`