-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathserver.ts
More file actions
179 lines (164 loc) · 7.78 KB
/
Copy pathserver.ts
File metadata and controls
179 lines (164 loc) · 7.78 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
import express from 'express';
import path from 'path';
import dotenv from 'dotenv';
import { createServer as createViteServer } from 'vite';
import { GoogleGenAI, Type } from '@google/genai';
dotenv.config();
const app = express();
const PORT = 3000;
app.use(express.json());
// Initialize Gemini SDK with telemetry header
const getGeminiClient = () => {
const apiKey = process.env.GEMINI_API_KEY;
if (!apiKey) {
throw new Error('GEMINI_API_KEY environment variable is missing.');
}
return new GoogleGenAI({
apiKey,
httpOptions: {
headers: {
'User-Agent': 'aistudio-build',
},
},
});
};
// API Route: Generate Sound Recipe from Text Description
app.post('/api/generate-sound', async (req, res) => {
try {
const { prompt, category } = req.body;
if (!prompt || typeof prompt !== 'string') {
res.status(400).json({ error: 'Prompt string is required' });
return;
}
const ai = getGeminiClient();
const systemInstruction = `You are a Master Sound Designer specializing in soft, warm, calm, organic UI sound effects for mobile language-learning apps (similar to Duolingo, Calm, and Headspace).
Your job is to translate a user's textual sound description into precise Web Audio synthesis parameters for short UI audio cues (under 1.0 second in duration).
IMPORTANT SOUND DESIGN PRINCIPLES FOR CALM UI SOUNDS:
1. Warmth over harshness: Cutoff frequencies should be smooth (typically 800Hz - 4500Hz).
2. Harmonic purity: Fundamental tone should be rich with gentle decay overtones (warm bell, wooden marimba, zen singing bowl, soft chime chord).
3. Soft attack: Avoid instant sharp clicks unless specifically requesting a subtle wooden tap. Use 0.008s - 0.04s attack.
4. Total duration: Must be short, under 1 second (typically 0.25s to 0.75s).
5. Chord structure: For success/unlock chimes, use pleasant harmonious ratios (e.g. Major 3rd, Perfect 5th, Octave, or Major 7th).
Output a structured JSON recipe matching the schema.`;
const response = await ai.models.generateContent({
model: 'gemini-3.6-flash',
contents: `Design a short, warm, soothing UI sound effect based on this description: "${prompt}". Category preference: ${category || 'general'}.`,
config: {
systemInstruction,
responseMimeType: 'application/json',
responseSchema: {
type: Type.OBJECT,
properties: {
title: { type: Type.STRING, description: 'Creative 2-4 word title for this sound effect' },
description: { type: Type.STRING, description: 'Sensory description of how this sound feels' },
suggestedEventType: {
type: Type.STRING,
description: 'Matching app event: correct, incorrect, tap, chapter_unlock, stroke_complete, or general',
},
durationSec: { type: Type.NUMBER, description: 'Duration in seconds between 0.1 and 0.95' },
recipe: {
type: Type.OBJECT,
properties: {
archetype: {
type: Type.STRING,
description: 'Archetype: bell, marimba, singing_bowl, wood_click, chime_chord, soft_sweep',
},
basePitchHz: { type: Type.NUMBER, description: 'Fundamental frequency in Hz (e.g. 261.63, 440, 523.25, 659.25)' },
chimeRatios: {
type: Type.ARRAY,
items: { type: Type.NUMBER },
description: 'Frequency multipliers for chords/harmonics (e.g. [1.0, 1.25, 1.5] for major chord, or [1.0] for single note)',
},
envelope: {
type: Type.OBJECT,
properties: {
attack: { type: Type.NUMBER, description: 'Attack time in seconds (0.005 to 0.1)' },
decay: { type: Type.NUMBER, description: 'Decay time in seconds (0.05 to 0.8)' },
sustain: { type: Type.NUMBER, description: 'Sustain level 0.0 to 0.15' },
release: { type: Type.NUMBER, description: 'Release time in seconds (0.05 to 0.4)' },
curve: { type: Type.STRING, description: 'exponential or linear' },
},
required: ['attack', 'decay', 'sustain', 'release', 'curve'],
},
timbre: {
type: Type.OBJECT,
properties: {
harmonicWeights: {
type: Type.ARRAY,
items: { type: Type.NUMBER },
description: 'Overtone relative volumes [fund, 2nd, 3rd, 4th, 5th] e.g. [1.0, 0.35, 0.1, 0.03]',
},
fmModulationIndex: { type: Type.NUMBER, description: 'FM index 0.0 to 2.5 for bell metallic resonance' },
fmFrequencyRatio: { type: Type.NUMBER, description: 'FM ratio relative to base pitch e.g. 1.414, 2.0, 3.5' },
},
required: ['harmonicWeights', 'fmModulationIndex', 'fmFrequencyRatio'],
},
filter: {
type: Type.OBJECT,
properties: {
cutoffHz: { type: Type.NUMBER, description: 'Lowpass filter cutoff in Hz (400 to 5000)' },
resonanceQ: { type: Type.NUMBER, description: 'Filter resonance Q (0.5 to 3.0)' },
sweepAmount: { type: Type.NUMBER, description: 'Filter sweep down in Hz (0 to 1200)' },
},
required: ['cutoffHz', 'resonanceQ', 'sweepAmount'],
},
warmth: {
type: Type.OBJECT,
properties: {
tapeSat: { type: Type.NUMBER, description: 'Soft saturation amount (0.0 to 0.35)' },
lowBoostDb: { type: Type.NUMBER, description: 'Warmth bass/mid boost in dB (0 to 4.0)' },
},
required: ['tapeSat', 'lowBoostDb'],
},
shimmer: {
type: Type.OBJECT,
properties: {
vibratoRate: { type: Type.NUMBER, description: 'Vibrato frequency Hz (0 to 7)' },
vibratoDepthCents: { type: Type.NUMBER, description: 'Vibrato depth in pitch cents (0 to 12)' },
pitchBendCents: { type: Type.NUMBER, description: 'Initial pitch bend in cents (-300 to +300)' },
},
required: ['vibratoRate', 'vibratoDepthCents', 'pitchBendCents'],
},
},
required: ['archetype', 'basePitchHz', 'chimeRatios', 'envelope', 'timbre', 'filter', 'warmth', 'shimmer'],
},
},
required: ['title', 'description', 'suggestedEventType', 'durationSec', 'recipe'],
},
},
});
const textResult = response.text || '';
const soundData = JSON.parse(textResult);
res.json({ success: true, data: soundData });
} catch (error: any) {
console.error('Error generating sound recipe with Gemini:', error);
res.status(500).json({
error: 'Failed to generate sound recipe from Gemini AI.',
details: error?.message || String(error),
});
}
});
// API Route: Health check
app.get('/api/health', (_req, res) => {
res.json({ status: 'ok', time: new Date().toISOString() });
});
// Start Express + Vite server
async function start() {
if (process.env.NODE_ENV !== 'production') {
const vite = await createViteServer({
server: { middlewareMode: true },
appType: 'spa',
});
app.use(vite.middlewares);
} else {
const distPath = path.join(process.cwd(), 'dist');
app.use(express.static(distPath));
app.get('*', (_req, res) => {
res.sendFile(path.join(distPath, 'index.html'));
});
}
app.listen(PORT, '0.0.0.0', () => {
console.log(`Server listening on http://0.0.0.0:${PORT}`);
});
}
start();