init

nodejs/README.md

@@ -0,0 +1,102 @@
+# TTS ONNX Node.js Implementation
+
+Node.js implementation for TTS inference. Uses ONNX Runtime to generate speech from text.
+
+## Requirements
+
+- Node.js v16 or higher
+- npm or yarn
+
+## Installation
+
+```bash
+cd nodejs
+npm install
+```
+
+## Basic Usage
+
+### Example 1: Default Inference
+Run inference with default settings:
+```bash
+npm start
+```
+
+Or:
+```bash
+node example_onnx.js
+```
+
+This will use:
+- Voice style: `assets/voice_styles/M1.json`
+- Text: "This morning, I took a walk in the park, and the sound of the birds and the breeze was so pleasant that I stopped for a long time just to listen."
+- Output directory: `results/`
+- Total steps: 5
+- Number of generations: 4
+
+### Example 2: Batch Inference
+Process multiple voice styles and texts at once:
+```bash
+node example_onnx.js \
+  --voice-style "assets/voice_styles/M1.json,assets/voice_styles/F1.json" \
+  --text "The sun sets behind the mountains, painting the sky in shades of pink and orange.|The weather is beautiful and sunny outside. A gentle breeze makes the air feel fresh and pleasant."
+```
+
+This will:
+- Generate speech for 2 different voice-text pairs
+- Use male voice style (M1.json) for the first text
+- Use female voice style (F1.json) for the second text
+- Process both samples in a single batch
+
+### Example 3: High Quality Inference
+Increase denoising steps for better quality:
+```bash
+node example_onnx.js \
+  --total-step 10 \
+  --voice-style "assets/voice_styles/M1.json" \
+  --text "Increasing the number of denoising steps improves the output's fidelity and overall quality."
+```
+
+This will:
+- Use 10 denoising steps instead of the default 5
+- Produce higher quality output at the cost of slower inference
+
+## Available Arguments
+
+| Argument | Type | Default | Description |
+|----------|------|---------|-------------|
+| `--use-gpu` | flag | False | Use GPU for inference (not supported yet) |
+| `--onnx-dir` | str | `assets/onnx` | Path to ONNX model directory |
+| `--total-step` | int | 5 | Number of denoising steps (higher = better quality, slower) |
+| `--n-test` | int | 4 | Number of times to generate each sample |
+| `--voice-style` | str+ | `assets/voice_styles/M1.json` | Voice style file path(s). Separate multiple files with commas |
+| `--text` | str+ | (long default text) | Text(s) to synthesize. Separate multiple texts with pipes |
+| `--save-dir` | str | `results` | Output directory |
+
+## Notes
+
+- **Batch Processing**: The number of voice style files must match the number of texts. Use commas to separate files and pipes to separate texts
+- **Quality vs Speed**: Higher `--total-step` values produce better quality but take longer
+- **GPU Support**: GPU mode is not supported yet
+
+## Architecture
+
+- `helper.js`: Node.js port of Python's `helper.py`
+  - `Preprocessor`: Audio preprocessing (STFT, Mel Spectrogram)
+  - `UnicodeProcessor`: Text preprocessing
+  - Utility functions (mask generation, tensor conversion, etc.)
+
+- `example_onnx.js`: Main inference script
+  - ONNX model loading
+  - TTS inference pipeline execution
+  - WAV file saving
+
+- `package.json`: Node.js project configuration and dependencies
+
+## Implementation Notes
+
+1. **Pure Node.js WAV Processing**: Writes WAV files without external native libraries. Outputs 16-bit PCM format.
+
+2. **Memory Efficiency**: Note that Node.js may consume significant memory when processing large arrays.
+
+3. **Performance**: The mel spectrogram extraction (Step 1-1) is currently slower than Python's Librosa, which uses highly optimized C extensions. This bottleneck could be further improved with additional optimizations such as WASM-based FFT libraries or native addons.

nodejs/assets

@@ -0,0 +1 @@
+../assets

nodejs/example_onnx.js

@@ -0,0 +1,104 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { loadTextToSpeech, loadVoiceStyle, timer, writeWavFile } from './helper.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Parse command line arguments
+ */
+function parseArgs() {
+    const args = {
+        useGpu: false,
+        onnxDir: 'assets/onnx',
+        totalStep: 5,
+        nTest: 4,
+        voiceStyle: ['assets/voice_styles/M1.json'],
+        text: ['This morning, I took a walk in the park, and the sound of the birds and the breeze was so pleasant that I stopped for a long time just to listen.'],
+        saveDir: 'results'
+    };
+
+    for (let i = 2; i < process.argv.length; i++) {
+        const arg = process.argv[i];
+        if (arg === '--use-gpu') {
+            args.useGpu = true;
+        } else if (arg === '--onnx-dir' && i + 1 < process.argv.length) {
+            args.onnxDir = process.argv[++i];
+        } else if (arg === '--total-step' && i + 1 < process.argv.length) {
+            args.totalStep = parseInt(process.argv[++i]);
+        } else if (arg === '--n-test' && i + 1 < process.argv.length) {
+            args.nTest = parseInt(process.argv[++i]);
+        } else if (arg === '--voice-style' && i + 1 < process.argv.length) {
+            args.voiceStyle = process.argv[++i].split(',');
+        } else if (arg === '--text' && i + 1 < process.argv.length) {
+            args.text = process.argv[++i].split('|');
+        } else if (arg === '--save-dir' && i + 1 < process.argv.length) {
+            args.saveDir = process.argv[++i];
+        }
+    }
+
+    return args;
+}
+
+/**
+ * Main inference function
+ */
+async function main() {
+    console.log('=== TTS Inference with ONNX Runtime (Node.js) ===\n');
+
+    // --- 1. Parse arguments --- //
+    const args = parseArgs();
+    const totalStep = args.totalStep;
+    const nTest = args.nTest;
+    const saveDir = args.saveDir;
+    const voiceStylePaths = args.voiceStyle.map(p => path.resolve(__dirname, p));
+    const textList = args.text;
+
+    if (voiceStylePaths.length !== textList.length) {
+        throw new Error(`Number of voice styles (${voiceStylePaths.length}) must match number of texts (${textList.length})`);
+    }
+
+    const bsz = voiceStylePaths.length;
+
+    // --- 2. Load Text to Speech --- //
+    const onnxDir = path.resolve(__dirname, args.onnxDir);
+    const textToSpeech = await loadTextToSpeech(onnxDir, args.useGpu);
+
+    // --- 3. Load Voice Style --- //
+    const style = loadVoiceStyle(voiceStylePaths, true);
+
+    // --- 4. Synthesize speech --- //
+    for (let n = 0; n < nTest; n++) {
+        console.log(`\n[${n + 1}/${nTest}] Starting synthesis...`);
+        
+        const { wav, duration } = await timer('Generating speech from text', async () => {
+            return await textToSpeech.call(textList, style, totalStep);
+        });
+        
+        if (!fs.existsSync(saveDir)) {
+            fs.mkdirSync(saveDir, { recursive: true });
+        }
+
+        const wavShape = [bsz, wav.length / bsz];
+        for (let b = 0; b < bsz; b++) {
+            const fname = `${textList[b].substring(0, 20).replace(/[^a-zA-Z0-9]/g, '_')}_${n + 1}.wav`;
+            const wavLen = Math.floor(textToSpeech.sampleRate * duration[b]);
+            const wavOut = wav.slice(b * wavShape[1], b * wavShape[1] + wavLen);
+            
+            const outputPath = path.join(saveDir, fname);
+            writeWavFile(outputPath, wavOut, textToSpeech.sampleRate);
+            console.log(`Saved: ${outputPath}`);
+        }
+    }
+
+    console.log('\n=== Synthesis completed successfully! ===');
+}
+
+// Run main function
+main().catch(err => {
+    console.error('Error during inference:', err);
+    process.exit(1);
+});

nodejs/helper.js

@@ -0,0 +1,392 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import * as ort from 'onnxruntime-node';
+
+const __filename = fileURLToPath(import.meta.url);
+
+/**
+ * Unicode text processor
+ */
+class UnicodeProcessor {
+    constructor(unicodeIndexerJsonPath) {
+        this.indexer = JSON.parse(fs.readFileSync(unicodeIndexerJsonPath, 'utf8'));
+    }
+
+    _preprocessText(text) {
+        // Simple NFKD normalization (JavaScript has normalize built-in)
+        return text.normalize('NFKD');
+    }
+
+    _textToUnicodeValues(text) {
+        return Array.from(text).map(char => char.charCodeAt(0));
+    }
+
+    _getTextMask(textIdsLengths) {
+        return lengthToMask(textIdsLengths);
+    }
+
+    call(textList) {
+        const processedTexts = textList.map(t => this._preprocessText(t));
+        const textIdsLengths = processedTexts.map(t => t.length);
+        const maxLen = Math.max(...textIdsLengths);
+        
+        const textIds = [];
+        for (let i = 0; i < processedTexts.length; i++) {
+            const row = new Array(maxLen).fill(0);
+            const unicodeVals = this._textToUnicodeValues(processedTexts[i]);
+            for (let j = 0; j < unicodeVals.length; j++) {
+                row[j] = this.indexer[unicodeVals[j]];
+            }
+            textIds.push(row);
+        }
+        
+        const textMask = this._getTextMask(textIdsLengths);
+        return { textIds, textMask };
+    }
+}
+
+/**
+ * Style class
+ */
+class Style {
+    constructor(styleTtlOnnx, styleDpOnnx) {
+        this.ttl = styleTtlOnnx;
+        this.dp = styleDpOnnx;
+    }
+}
+
+/**
+ * TextToSpeech class
+ */
+class TextToSpeech {
+    constructor(cfgs, textProcessor, dpOrt, textEncOrt, vectorEstOrt, vocoderOrt) {
+        this.cfgs = cfgs;
+        this.textProcessor = textProcessor;
+        this.dpOrt = dpOrt;
+        this.textEncOrt = textEncOrt;
+        this.vectorEstOrt = vectorEstOrt;
+        this.vocoderOrt = vocoderOrt;
+        this.sampleRate = cfgs.ae.sample_rate;
+        this.baseChunkSize = cfgs.ae.base_chunk_size;
+        this.chunkCompressFactor = cfgs.ttl.chunk_compress_factor;
+        this.ldim = cfgs.ttl.latent_dim;
+    }
+
+    sampleNoisyLatent(duration) {
+        const wavLenMax = Math.max(...duration) * this.sampleRate;
+        const wavLengths = duration.map(d => Math.floor(d * this.sampleRate));
+        const chunkSize = this.baseChunkSize * this.chunkCompressFactor;
+        const latentLen = Math.floor((wavLenMax + chunkSize - 1) / chunkSize);
+        const latentDim = this.ldim * this.chunkCompressFactor;
+
+        // Generate random noise
+        const noisyLatent = [];
+        for (let b = 0; b < duration.length; b++) {
+            const batch = [];
+            for (let d = 0; d < latentDim; d++) {
+                const row = [];
+                for (let t = 0; t < latentLen; t++) {
+                    // Box-Muller transform for normal distribution
+                    // Add epsilon to avoid log(0)
+                    const eps = 1e-10;
+                    const u1 = Math.max(eps, Math.random());
+                    const u2 = Math.random();
+                    const randNormal = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2);
+                    row.push(randNormal);
+                }
+                batch.push(row);
+            }
+            noisyLatent.push(batch);
+        }
+
+        const latentMask = getLatentMask(wavLengths, this.baseChunkSize, this.chunkCompressFactor);
+        
+        // Apply mask
+        for (let b = 0; b < noisyLatent.length; b++) {
+            for (let d = 0; d < noisyLatent[b].length; d++) {
+                for (let t = 0; t < noisyLatent[b][d].length; t++) {
+                    noisyLatent[b][d][t] *= latentMask[b][0][t];
+                }
+            }
+        }
+
+        return { noisyLatent, latentMask };
+    }
+
+    async call(textList, style, totalStep) {
+        if (textList.length !== style.ttl.dims[0]) {
+            throw new Error('Number of texts must match number of style vectors');
+        }
+        const bsz = textList.length;
+        const { textIds, textMask } = this.textProcessor.call(textList);
+        const textIdsShape = [bsz, textIds[0].length];
+        const textMaskShape = [bsz, 1, textMask[0][0].length];
+        
+        const textMaskTensor = arrayToTensor(textMask, textMaskShape);
+        
+        const dpResult = await this.dpOrt.run({
+            text_ids: intArrayToTensor(textIds, textIdsShape),
+            style_dp: style.dp,
+            text_mask: textMaskTensor
+        });
+        
+        const durOnnx = Array.from(dpResult.duration.data);
+        
+        const textEncResult = await this.textEncOrt.run({
+            text_ids: intArrayToTensor(textIds, textIdsShape),
+            style_ttl: style.ttl,
+            text_mask: textMaskTensor
+        });
+        
+        const textEmbTensor = textEncResult.text_emb;
+
+        let { noisyLatent, latentMask } = this.sampleNoisyLatent(durOnnx);
+        const latentShape = [bsz, noisyLatent[0].length, noisyLatent[0][0].length];
+        const latentMaskShape = [bsz, 1, latentMask[0][0].length];
+        
+        const latentMaskTensor = arrayToTensor(latentMask, latentMaskShape);
+        
+        const totalStepArray = new Array(bsz).fill(totalStep);
+        const scalarShape = [bsz];
+        const totalStepTensor = arrayToTensor(totalStepArray, scalarShape);
+
+        for (let step = 0; step < totalStep; step++) {
+            const currentStepArray = new Array(bsz).fill(step);
+
+            const vectorEstResult = await this.vectorEstOrt.run({
+                noisy_latent: arrayToTensor(noisyLatent, latentShape),
+                text_emb: textEmbTensor,
+                style_ttl: style.ttl,
+                text_mask: textMaskTensor,
+                latent_mask: latentMaskTensor,
+                total_step: totalStepTensor,
+                current_step: arrayToTensor(currentStepArray, scalarShape)
+            });
+
+            const denoisedLatent = Array.from(vectorEstResult.denoised_latent.data);
+
+            // Update latent with the denoised output
+            let idx = 0;
+            for (let b = 0; b < noisyLatent.length; b++) {
+                for (let d = 0; d < noisyLatent[b].length; d++) {
+                    for (let t = 0; t < noisyLatent[b][d].length; t++) {
+                        noisyLatent[b][d][t] = denoisedLatent[idx++];
+                    }
+                }
+            }
+        }
+
+        const vocoderResult = await this.vocoderOrt.run({
+            latent: arrayToTensor(noisyLatent, latentShape)
+        });
+
+        const wav = Array.from(vocoderResult.wav_tts.data);
+        return { wav, duration: durOnnx };
+    }
+}
+
+/**
+ * Convert lengths to binary mask
+ */
+function lengthToMask(lengths, maxLen = null) {
+    maxLen = maxLen || Math.max(...lengths);
+    const mask = [];
+    for (let i = 0; i < lengths.length; i++) {
+        const row = [];
+        for (let j = 0; j < maxLen; j++) {
+            row.push(j < lengths[i] ? 1.0 : 0.0);
+        }
+        mask.push([row]); // [B, 1, maxLen]
+    }
+    return mask;
+}
+
+/**
+ * Get latent mask from wav lengths
+ */
+function getLatentMask(wavLengths, baseChunkSize, chunkCompressFactor) {
+    const latentSize = baseChunkSize * chunkCompressFactor;
+    const latentLengths = wavLengths.map(len => 
+        Math.floor((len + latentSize - 1) / latentSize)
+    );
+    return lengthToMask(latentLengths);
+}
+
+/**
+ * Load ONNX model
+ */
+async function loadOnnx(onnxPath, opts) {
+    return await ort.InferenceSession.create(onnxPath, opts);
+}
+
+/**
+ * Load all ONNX models for TTS
+ */
+async function loadOnnxAll(onnxDir, opts) {
+    const dpPath = path.join(onnxDir, 'duration_predictor.onnx');
+    const textEncPath = path.join(onnxDir, 'text_encoder.onnx');
+    const vectorEstPath = path.join(onnxDir, 'vector_estimator.onnx');
+    const vocoderPath = path.join(onnxDir, 'vocoder.onnx');
+
+    const [dpOrt, textEncOrt, vectorEstOrt, vocoderOrt] = await Promise.all([
+        loadOnnx(dpPath, opts),
+        loadOnnx(textEncPath, opts),
+        loadOnnx(vectorEstPath, opts),
+        loadOnnx(vocoderPath, opts)
+    ]);
+
+    return { dpOrt, textEncOrt, vectorEstOrt, vocoderOrt };
+}
+
+/**
+ * Load configuration
+ */
+function loadCfgs(onnxDir) {
+    const cfgPath = path.join(onnxDir, 'tts.json');
+    const cfgs = JSON.parse(fs.readFileSync(cfgPath, 'utf8'));
+    return cfgs;
+}
+
+/**
+ * Load text processor
+ */
+function loadTextProcessor(onnxDir) {
+    const unicodeIndexerPath = path.join(onnxDir, 'unicode_indexer.json');
+    const textProcessor = new UnicodeProcessor(unicodeIndexerPath);
+    return textProcessor;
+}
+
+/**
+ * Load voice style from JSON file
+ */
+export function loadVoiceStyle(voiceStylePaths, verbose = false) {
+    const bsz = voiceStylePaths.length;
+    
+    // Read first file to get dimensions
+    const firstStyle = JSON.parse(fs.readFileSync(voiceStylePaths[0], 'utf8'));
+    const ttlDims = firstStyle.style_ttl.dims;
+    const dpDims = firstStyle.style_dp.dims;
+    
+    const ttlDim1 = ttlDims[1];
+    const ttlDim2 = ttlDims[2];
+    const dpDim1 = dpDims[1];
+    const dpDim2 = dpDims[2];
+    
+    // Pre-allocate arrays with full batch size
+    const ttlSize = bsz * ttlDim1 * ttlDim2;
+    const dpSize = bsz * dpDim1 * dpDim2;
+    const ttlFlat = new Float32Array(ttlSize);
+    const dpFlat = new Float32Array(dpSize);
+    
+    // Fill in the data
+    for (let i = 0; i < bsz; i++) {
+        const voiceStyle = JSON.parse(fs.readFileSync(voiceStylePaths[i], 'utf8'));
+        
+        const ttlData = voiceStyle.style_ttl.data.flat(Infinity);
+        const ttlOffset = i * ttlDim1 * ttlDim2;
+        ttlFlat.set(ttlData, ttlOffset);
+        
+        const dpData = voiceStyle.style_dp.data.flat(Infinity);
+        const dpOffset = i * dpDim1 * dpDim2;
+        dpFlat.set(dpData, dpOffset);
+    }
+    
+    const ttlStyle = new ort.Tensor('float32', ttlFlat, [bsz, ttlDim1, ttlDim2]);
+    const dpStyle = new ort.Tensor('float32', dpFlat, [bsz, dpDim1, dpDim2]);
+    
+    if (verbose) {
+        console.log(`Loaded ${bsz} voice styles`);
+    }
+    
+    return new Style(ttlStyle, dpStyle);
+}
+
+/**
+ * Load text to speech components
+ */
+export async function loadTextToSpeech(onnxDir, useGpu = false) {
+    const opts = {};
+    if (useGpu) {
+        throw new Error('GPU mode is not supported yet');
+    } else {
+        console.log('Using CPU for inference');
+    }
+    
+    const cfgs = loadCfgs(onnxDir);
+    const { dpOrt, textEncOrt, vectorEstOrt, vocoderOrt } = await loadOnnxAll(onnxDir, opts);
+    const textProcessor = loadTextProcessor(onnxDir);
+    const textToSpeech = new TextToSpeech(cfgs, textProcessor, dpOrt, textEncOrt, vectorEstOrt, vocoderOrt);
+    
+    return textToSpeech;
+}
+
+/**
+ * Convert 3D array to ONNX tensor
+ */
+function arrayToTensor(array, dims) {
+    // Flatten the array
+    const flat = array.flat(Infinity);
+    return new ort.Tensor('float32', Float32Array.from(flat), dims);
+}
+
+/**
+ * Convert 2D int array to ONNX tensor
+ */
+function intArrayToTensor(array, dims) {
+    const flat = array.flat(Infinity);
+    return new ort.Tensor('int64', BigInt64Array.from(flat.map(x => BigInt(x))), dims);
+}
+
+/**
+ * Write WAV file
+ */
+export function writeWavFile(filename, audioData, sampleRate) {
+    const numChannels = 1;
+    const bitsPerSample = 16;
+    const byteRate = sampleRate * numChannels * bitsPerSample / 8;
+    const blockAlign = numChannels * bitsPerSample / 8;
+    const dataSize = audioData.length * bitsPerSample / 8;
+
+    const buffer = Buffer.alloc(44 + dataSize);
+    
+    // RIFF header
+    buffer.write('RIFF', 0);
+    buffer.writeUInt32LE(36 + dataSize, 4);
+    buffer.write('WAVE', 8);
+    
+    // fmt chunk
+    buffer.write('fmt ', 12);
+    buffer.writeUInt32LE(16, 16); // fmt chunk size
+    buffer.writeUInt16LE(1, 20); // audio format (PCM)
+    buffer.writeUInt16LE(numChannels, 22);
+    buffer.writeUInt32LE(sampleRate, 24);
+    buffer.writeUInt32LE(byteRate, 28);
+    buffer.writeUInt16LE(blockAlign, 32);
+    buffer.writeUInt16LE(bitsPerSample, 34);
+    
+    // data chunk
+    buffer.write('data', 36);
+    buffer.writeUInt32LE(dataSize, 40);
+    
+    // Write audio data
+    for (let i = 0; i < audioData.length; i++) {
+        const sample = Math.max(-1, Math.min(1, audioData[i]));
+        const intSample = Math.floor(sample * 32767);
+        buffer.writeInt16LE(intSample, 44 + i * 2);
+    }
+    
+    fs.writeFileSync(filename, buffer);
+}
+
+/**
+ * Timer utility for measuring execution time
+ */
+export async function timer(name, fn) {
+    const start = Date.now();
+    console.log(`${name}...`);
+    const result = await fn();
+    const elapsed = ((Date.now() - start) / 1000).toFixed(2);
+    console.log(`  -> ${name} completed in ${elapsed} sec`);
+    return result;
+}

nodejs/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "tts-onnx-nodejs",
+  "version": "1.0.0",
+  "description": "TTS inference using ONNX Runtime for Node.js",
+  "main": "example_onnx.js",
+  "type": "module",
+  "scripts": {
+    "start": "node example_onnx.js"
+  },
+  "keywords": [
+    "tts",
+    "onnx",
+    "speech-synthesis",
+    "nodejs"
+  ],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "fft.js": "^4.0.3",
+    "js-yaml": "^4.1.0",
+    "onnxruntime-node": "^1.19.2"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  }
+}