Supertonic 2

2026-01-06 17:15:20 +09:00
parent ceedbbb835
commit e71c516714
44 changed files with 1105 additions and 581 deletions
--- a/java/ExampleONNX.java
+++ b/java/ExampleONNX.java
@@ -21,6 +21,7 @@ public class ExampleONNX {
        List<String> text = Arrays.asList(
            "This morning, I took a walk in the park, and the sound of the birds and the breeze was so pleasant that I stopped for a long time just to listen."
        );
+        List<String> lang = Arrays.asList("en");
        String saveDir = "results";
        boolean batch = false;
    }
@@ -58,6 +59,11 @@ public class ExampleONNX {
                        result.text = Arrays.asList(args[++i].split("\\|"));
                    }
                    break;
+                case "--lang":
+                    if (i + 1 < args.length) {
+                        result.lang = Arrays.asList(args[++i].split(","));
+                    }
+                    break;
                case "--save-dir":
                    if (i + 1 < args.length) result.saveDir = args[++i];
                    break;
@@ -85,6 +91,7 @@ public class ExampleONNX {
            String saveDir = parsedArgs.saveDir;
            List<String> voiceStylePaths = parsedArgs.voiceStyle;
            List<String> textList = parsedArgs.text;
+            List<String> langList = parsedArgs.lang;
            boolean batch = parsedArgs.batch;
            
            if (batch) {
@@ -92,6 +99,10 @@ public class ExampleONNX {
                    throw new RuntimeException("Number of voice styles (" + voiceStylePaths.size() + 
                        ") must match number of texts (" + textList.size() + ")");
                }
+                if (langList.size() != textList.size()) {
+                    throw new RuntimeException("Number of languages (" + langList.size() + 
+                        ") must match number of texts (" + textList.size() + ")");
+                }
            }
            
            int bsz = voiceStylePaths.size();
@@ -116,7 +127,7 @@ public class ExampleONNX {
                if (batch) {
                    ttsResult = Helper.timer("Generating speech from text", () -> {
                        try {
-                            return textToSpeech.batch(textList, style, totalStep, speed, env);
+                            return textToSpeech.batch(textList, langList, style, totalStep, speed, env);
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
@@ -124,7 +135,7 @@ public class ExampleONNX {
                } else {
                    ttsResult = Helper.timer("Generating speech from text", () -> {
                        try {
-                            return textToSpeech.call(textList.get(0), style, totalStep, speed, 0.3f, env);
+                            return textToSpeech.call(textList.get(0), langList.get(0), style, totalStep, speed, 0.3f, env);
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
--- a/java/Helper.java
+++ b/java/Helper.java
@@ -18,6 +18,17 @@ import java.util.*;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;

+/**
+ * Available languages for multilingual TTS
+ */
+class Languages {
+    public static final List<String> AVAILABLE = Arrays.asList("en", "ko", "es", "pt", "fr");
+    
+    public static boolean isValid(String lang) {
+        return AVAILABLE.contains(lang);
+    }
+}
+
 /**
 * Configuration classes
 */
@@ -96,22 +107,28 @@ class UnicodeProcessor {
        return result.toString();
    }
    
-    public TextProcessResult call(List<String> textList) {
+    public TextProcessResult call(List<String> textList, List<String> langList) {
        List<String> processedTexts = new ArrayList<>();
-        for (String text : textList) {
-            processedTexts.add(preprocessText(text));
+        for (int i = 0; i < textList.size(); i++) {
+            processedTexts.add(preprocessText(textList.get(i), langList.get(i)));
+        }
+        
+        // Convert texts to unicode values first to get correct character counts
+        List<int[]> allUnicodeVals = new ArrayList<>();
+        for (String text : processedTexts) {
+            allUnicodeVals.add(textToUnicodeValues(text));
        }
        
        int[] textIdsLengths = new int[processedTexts.size()];
        int maxLen = 0;
-        for (int i = 0; i < processedTexts.size(); i++) {
-            textIdsLengths[i] = processedTexts.get(i).length();
+        for (int i = 0; i < allUnicodeVals.size(); i++) {
+            textIdsLengths[i] = allUnicodeVals.get(i).length;  // Use code point count, not char count
            maxLen = Math.max(maxLen, textIdsLengths[i]);
        }
        
        long[][] textIds = new long[processedTexts.size()][maxLen];
-        for (int i = 0; i < processedTexts.size(); i++) {
-            int[] unicodeVals = textToUnicodeValues(processedTexts.get(i));
+        for (int i = 0; i < allUnicodeVals.size(); i++) {
+            int[] unicodeVals = allUnicodeVals.get(i);
            for (int j = 0; j < unicodeVals.length; j++) {
                textIds[i][j] = indexer[unicodeVals[j]];
            }
@@ -121,12 +138,10 @@ class UnicodeProcessor {
        return new TextProcessResult(textIds, textMask);
    }
    
-    private String preprocessText(String text) {
+    private String preprocessText(String text, String lang) {
        // TODO: Need advanced normalizer for better performance
        text = Normalizer.normalize(text, Normalizer.Form.NFKD);

-        // FIXME: this should be fixed for non-English languages
-
        // Remove emojis (wide Unicode range)
        // Java Pattern doesn't support \x{...} syntax for Unicode above \uFFFF
        // Use character filtering instead
@@ -137,7 +152,6 @@ class UnicodeProcessor {
        replacements.put("–", "-");      // en dash
        replacements.put("‑", "-");      // non-breaking hyphen
        replacements.put("—", "-");      // em dash
-        replacements.put("¯", " ");      // macron
        replacements.put("_", " ");      // underscore
        replacements.put("\u201C", "\"");     // left double quote
        replacements.put("\u201D", "\"");     // right double quote
@@ -157,9 +171,6 @@ class UnicodeProcessor {
            text = text.replace(entry.getKey(), entry.getValue());
        }

-        // Remove combining diacritics // FIXME: this should be fixed for non-English languages
-        text = text.replaceAll("[\\u0302\\u0303\\u0304\\u0305\\u0306\\u0307\\u0308\\u030A\\u030B\\u030C\\u0327\\u0328\\u0329\\u032A\\u032B\\u032C\\u032D\\u032E\\u032F]", "");
-
        // Remove special symbols
        text = text.replaceAll("[♥☆♡©\\\\]", "");

@@ -201,15 +212,20 @@ class UnicodeProcessor {
            text += ".";
        }

+        // Validate language
+        if (!Languages.isValid(lang)) {
+            throw new IllegalArgumentException("Invalid language: " + lang + ". Available: " + Languages.AVAILABLE);
+        }
+
+        // Wrap text with language tags
+        text = "<" + lang + ">" + text + "</" + lang + ">";
+
        return text;
    }
    
    private int[] textToUnicodeValues(String text) {
-        int[] values = new int[text.length()];
-        for (int i = 0; i < text.length(); i++) {
-            values[i] = text.codePointAt(i);
-        }
-        return values;
+        // Use codePoints() stream to correctly handle surrogate pairs
+        return text.codePoints().toArray();
    }
    
    private float[][][] getTextMask(int[] lengths) {
@@ -269,12 +285,12 @@ class TextToSpeech {
        this.ldim = config.ttl.latentDim;
    }
    
-    private TTSResult _infer(List<String> textList, Style style, int totalStep, float speed, OrtEnvironment env) 
+    private TTSResult _infer(List<String> textList, List<String> langList, Style style, int totalStep, float speed, OrtEnvironment env) 
            throws OrtException {
        int bsz = textList.size();
        
        // Process text
-        UnicodeProcessor.TextProcessResult textResult = textProcessor.call(textList);
+        UnicodeProcessor.TextProcessResult textResult = textProcessor.call(textList, langList);
        long[][] textIds = textResult.textIds;
        float[][][] textMask = textResult.textMask;
        
@@ -360,7 +376,18 @@ class TextToSpeech {
        
        OrtSession.Result vocoderResult = vocoderSession.run(vocoderInputs);
        float[][] wavBatch = (float[][]) vocoderResult.get(0).getValue();
-        float[] wav = wavBatch[0];
+        
+        // Flatten all batch audio into a single array for batch processing
+        int totalSamples = 0;
+        for (float[] w : wavBatch) {
+            totalSamples += w.length;
+        }
+        float[] wav = new float[totalSamples];
+        int offset = 0;
+        for (float[] w : wavBatch) {
+            System.arraycopy(w, 0, wav, offset, w.length);
+            offset += w.length;
+        }
        
        // Clean up
        textIdsTensor.close();
@@ -421,15 +448,16 @@ class TextToSpeech {
    /**
     * Synthesize speech from a single text with automatic chunking
     */
-    public TTSResult call(String text, Style style, int totalStep, float speed, float silenceDuration, OrtEnvironment env) 
+    public TTSResult call(String text, String lang, Style style, int totalStep, float speed, float silenceDuration, OrtEnvironment env) 
            throws OrtException {
-        List<String> chunks = Helper.chunkText(text, 0);
+        int maxLen = lang.equals("ko") ? 120 : 300;
+        List<String> chunks = Helper.chunkText(text, maxLen);
        
        List<Float> wavCat = new ArrayList<>();
        float durCat = 0.0f;
        
        for (int i = 0; i < chunks.size(); i++) {
-            TTSResult result = _infer(Arrays.asList(chunks.get(i)), style, totalStep, speed, env);
+            TTSResult result = _infer(Arrays.asList(chunks.get(i)), Arrays.asList(lang), style, totalStep, speed, env);
            
            float dur = result.duration[0];
            int wavLen = (int) (sampleRate * dur);
@@ -464,9 +492,9 @@ class TextToSpeech {
    /**
     * Batch synthesize speech from multiple texts
     */
-    public TTSResult batch(List<String> textList, Style style, int totalStep, float speed, OrtEnvironment env) 
+    public TTSResult batch(List<String> textList, List<String> langList, Style style, int totalStep, float speed, OrtEnvironment env) 
            throws OrtException {
-        return _infer(textList, style, totalStep, speed, env);
+        return _infer(textList, langList, style, totalStep, speed, env);
    }
    
    public void close() throws OrtException {
@@ -841,13 +869,20 @@ public class Helper {
    }
    
    /**
-     * Sanitize filename
+     * Sanitize filename (supports Unicode characters)
     */
    public static String sanitizeFilename(String text, int maxLen) {
-        if (text.length() > maxLen) {
-            text = text.substring(0, maxLen);
+        // Get first maxLen characters (code points, not chars for surrogate pairs)
+        int[] codePoints = text.codePoints().limit(maxLen).toArray();
+        StringBuilder result = new StringBuilder();
+        for (int codePoint : codePoints) {
+            if (Character.isLetterOrDigit(codePoint)) {
+                result.appendCodePoint(codePoint);
+            } else {
+                result.append('_');
+            }
        }
-        return text.replaceAll("[^a-zA-Z0-9]", "_");
+        return result.toString();
    }
    
    /**
--- a/java/README.md
+++ b/java/README.md
@@ -4,6 +4,8 @@ This guide provides examples for running TTS inference using `ExampleONNX.java`.

 ## 📰 Update News

+**2026.01.06** - 🎉 **Supertonic 2** released with multilingual support! Now supports English (`en`), Korean (`ko`), Spanish (`es`), Portuguese (`pt`), and French (`fr`). [Demo](https://huggingface.co/spaces/Supertone/supertonic-2) | [Models](https://huggingface.co/Supertone/supertonic-2)
+
 **2025.12.10** - Added [6 new voice styles](https://huggingface.co/Supertone/supertonic/tree/b10dbaf18b316159be75b34d24f740008fddd381) (M3, M4, M5, F3, F4, F5). See [Voices](https://supertone-inc.github.io/supertonic-py/voices/) for details

 **2025.12.08** - Optimized ONNX models via [OnnxSlim](https://github.com/inisis/OnnxSlim) now available on [Hugging Face Models](https://huggingface.co/Supertone/supertonic)
@@ -47,13 +49,13 @@ This will use:
 ### Example 2: Batch Inference
 Process multiple voice styles and texts at once:
 ```bash
-mvn exec:java -Dexec.args="--batch --voice-style assets/voice_styles/M1.json,assets/voice_styles/F1.json --text 'The sun sets behind the mountains, painting the sky in shades of pink and orange.|The weather is beautiful and sunny outside. A gentle breeze makes the air feel fresh and pleasant.'"
+mvn exec:java -Dexec.args="--batch --voice-style assets/voice_styles/M1.json,assets/voice_styles/F1.json --text 'The sun sets behind the mountains, painting the sky in shades of pink and orange.|오늘 아침에 공원을 산책했는데, 새소리와 바람 소리가 너무 기분 좋았어요.' --lang en,ko"
 ```

 This will:
- Generate speech for 2 different voice-text pairs
- Use male voice (M1.json) for the first text
- Use female voice (F1.json) for the second text
+- Generate speech for 2 different voice-text-language pairs
+- Use male voice (M1.json) for the first text in English
+- Use female voice (F1.json) for the second text in Korean
 - Process both samples in a single batch

 ### Example 3: High Quality Inference
@@ -111,14 +113,16 @@ java -jar target/tts-example.jar --total-step 10 --text "Your custom text here"
 | `--onnx-dir` | str | `assets/onnx` | Path to ONNX model directory |
 | `--total-step` | int | 5 | Number of denoising steps (higher = better quality, slower) |
 | `--n-test` | int | 4 | Number of times to generate each sample |
-| `--voice-style` | str+ | `assets/voice_styles/M1.json` | Voice style file path(s) |
-| `--text` | str+ | (long default text) | Text(s) to synthesize |
+| `--voice-style` | str+ | `assets/voice_styles/M1.json` | Voice style file path(s), comma-separated |
+| `--text` | str+ | (long default text) | Text(s) to synthesize, pipe-separated |
+| `--lang` | str+ | `en` | Language(s) for synthesis, comma-separated (en, ko, es, pt, fr) |
 | `--save-dir` | str | `results` | Output directory |
 | `--batch` | flag | False | Enable batch mode (multiple text-style pairs, disables automatic chunking) |

 ## Notes

- **Batch Processing**: When using `--batch`, the number of `--voice-style` files must match the number of `--text` entries
+- **Multilingual Support**: Use `--lang` to specify the language for each text. Available: `en` (English), `ko` (Korean), `es` (Spanish), `pt` (Portuguese), `fr` (French)
+- **Batch Processing**: When using `--batch`, the number of `--voice-style`, `--text`, and `--lang` entries must match
 - **Automatic Chunking**: Without `--batch`, long texts are automatically split and concatenated with 0.3s pauses
 - **Quality vs Speed**: Higher `--total-step` values produce better quality but take longer
 - **GPU Support**: GPU mode is not supported yet