diff --git a/etc/test_sentences/test_pt-br.jsonl b/etc/test_sentences/test_pt-br.jsonl index dfcad01..14b4f66 100644 --- a/etc/test_sentences/test_pt-br.jsonl +++ b/etc/test_sentences/test_pt-br.jsonl @@ -2,7 +2,7 @@ {"text": "É um arco multicolorido com o vermelho em seu exterior e o violeta em seu interior.", "phonemes": ["ɛ", " ", "u", "̃", "ŋ", " ", "ˈ", "a", "ɾ", "ə", "k", "ʊ", " ", "m", "ˌ", "u", "w", "t", "ʃ", "i", "k", "ˌ", "o", "l", "o", "ɾ", "ˈ", "i", "d", "ʊ", " ", "k", "o", "ŋ", " ", "ʊ", " ", "v", "ˌ", "e", "ɾ", "ə", "m", "ˈ", "e", "l", "j", "w", " ", "ˈ", "e", "ɪ", "ŋ", " ", "s", "e", "ʊ", " ", "ˌ", "e", "s", "t", "e", "ɾ", "i", "ˈ", "o", "ɾ", " ", "i", " ", "ʊ", " ", "v", "ˌ", "i", "o", "l", "ˈ", "e", "t", "æ", " ", "ˈ", "e", "ɪ", "ŋ", " ", "s", "e", "ʊ", " ", "ˌ", "i", "ŋ", "t", "e", "ɾ", "i", "ˈ", "o", "r", "."], "phoneme_ids": [1, 0, 61, 0, 3, 0, 33, 0, 141, 0, 44, 0, 3, 0, 120, 0, 14, 0, 92, 0, 59, 0, 23, 0, 100, 0, 3, 0, 25, 0, 121, 0, 33, 0, 35, 0, 32, 0, 96, 0, 21, 0, 23, 0, 121, 0, 27, 0, 24, 0, 27, 0, 92, 0, 120, 0, 21, 0, 17, 0, 100, 0, 3, 0, 23, 0, 27, 0, 44, 0, 3, 0, 100, 0, 3, 0, 34, 0, 121, 0, 18, 0, 92, 0, 59, 0, 25, 0, 120, 0, 18, 0, 24, 0, 22, 0, 35, 0, 3, 0, 120, 0, 18, 0, 74, 0, 44, 0, 3, 0, 31, 0, 18, 0, 100, 0, 3, 0, 121, 0, 18, 0, 31, 0, 32, 0, 18, 0, 92, 0, 21, 0, 120, 0, 27, 0, 92, 0, 3, 0, 21, 0, 3, 0, 100, 0, 3, 0, 34, 0, 121, 0, 21, 0, 27, 0, 24, 0, 120, 0, 18, 0, 32, 0, 39, 0, 3, 0, 120, 0, 18, 0, 74, 0, 44, 0, 3, 0, 31, 0, 18, 0, 100, 0, 3, 0, 121, 0, 21, 0, 44, 0, 32, 0, 18, 0, 92, 0, 21, 0, 120, 0, 27, 0, 30, 0, 10, 0, 2]} {"text": "Por ser um espectro de dispersão da luz branca, o arco-íris contém uma quantidade infinita de cores sem qualquer delimitação entre elas.", "phonemes": ["p", "o", "r", " ", "s", "e", "ɾ", " ", "u", "̃", "ŋ", " ", "ˌ", "e", "s", "p", "ˈ", "ɛ", "k", "t", "r", "ʊ", " ", "d", "ʒ", "y", " ", "d", "ʒ", "ˌ", "i", "s", "p", "e", "ɾ", "ə", "s", "ˈ", "ɐ", "̃", "ʊ", "̃", " ", "d", "a", " ", "l", "ˈ", "u", "z", " ", "b", "r", "ˈ", "ɐ", "̃", "ŋ", "k", "æ", ",", " ", "u", " ", "ˈ", "a", "ɾ", "ə", "k", "ʊ", "ˈ", "i", "ɾ", "i", "s", " ", "k", "o", "ŋ", "t", "ˈ", "e", "ɪ", "ŋ", " ", "ˌ", "u", "m", "æ", " ", "k", "w", "ˌ", "ɐ", "̃", "ŋ", "t", "ʃ", "i", "d", "ˈ", "a", "d", "ʒ", "j", " ", "ˌ", "i", "ŋ", "f", "i", "n", "ˈ", "i", "t", "æ", " ", "d", "ʒ", "y", " ", "k", "ˈ", "o", "ɾ", "y", "s", " ", "s", "ˈ", "e", "ɪ", "ŋ", " ", "k", "w", "a", "ʊ", "k", "ˌ", "ɛ", "r", " ", "d", "ˌ", "e", "l", "i", "m", "ˌ", "i", "t", "a", "s", "ˈ", "ɐ", "̃", "ʊ", "̃", " ", "ˈ", "e", "ɪ", "ŋ", "t", "r", "i", " ", "ˈ", "ɛ", "l", "æ", "s", "."], "phoneme_ids": [1, 0, 28, 0, 27, 0, 30, 0, 3, 0, 31, 0, 18, 0, 92, 0, 3, 0, 33, 0, 141, 0, 44, 0, 3, 0, 121, 0, 18, 0, 31, 0, 28, 0, 120, 0, 61, 0, 23, 0, 32, 0, 30, 0, 100, 0, 3, 0, 17, 0, 108, 0, 37, 0, 3, 0, 17, 0, 108, 0, 121, 0, 21, 0, 31, 0, 28, 0, 18, 0, 92, 0, 59, 0, 31, 0, 120, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 17, 0, 14, 0, 3, 0, 24, 0, 120, 0, 33, 0, 38, 0, 3, 0, 15, 0, 30, 0, 120, 0, 50, 0, 141, 0, 44, 0, 23, 0, 39, 0, 8, 0, 3, 0, 33, 0, 3, 0, 120, 0, 14, 0, 92, 0, 59, 0, 23, 0, 100, 0, 120, 0, 21, 0, 92, 0, 21, 0, 31, 0, 3, 0, 23, 0, 27, 0, 44, 0, 32, 0, 120, 0, 18, 0, 74, 0, 44, 0, 3, 0, 121, 0, 33, 0, 25, 0, 39, 0, 3, 0, 23, 0, 35, 0, 121, 0, 50, 0, 141, 0, 44, 0, 32, 0, 96, 0, 21, 0, 17, 0, 120, 0, 14, 0, 17, 0, 108, 0, 22, 0, 3, 0, 121, 0, 21, 0, 44, 0, 19, 0, 21, 0, 26, 0, 120, 0, 21, 0, 32, 0, 39, 0, 3, 0, 17, 0, 108, 0, 37, 0, 3, 0, 23, 0, 120, 0, 27, 0, 92, 0, 37, 0, 31, 0, 3, 0, 31, 0, 120, 0, 18, 0, 74, 0, 44, 0, 3, 0, 23, 0, 35, 0, 14, 0, 100, 0, 23, 0, 121, 0, 61, 0, 30, 0, 3, 0, 17, 0, 121, 0, 18, 0, 24, 0, 21, 0, 25, 0, 121, 0, 21, 0, 32, 0, 14, 0, 31, 0, 120, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 120, 0, 18, 0, 74, 0, 44, 0, 32, 0, 30, 0, 21, 0, 3, 0, 120, 0, 61, 0, 24, 0, 39, 0, 31, 0, 10, 0, 2]} {"text": "Devido à necessidade humana de classificação dos fenômenos da natureza, a capacidade finita de distinção de cores pela visão humana e por questões didáticas, o arco-íris é mais conhecido por uma simplificação criada culturalmente que resume o espectro em sete cores na seguinte ordem: vermelho, laranja, amarelo, verde, azul, anil e violeta.", "phonemes": ["d", "ˌ", "e", "v", "ˈ", "i", "d", "w", " ", "ˌ", "a", "ː", " ", "n", "ˌ", "e", "s", "e", "s", "i", "d", "ˈ", "a", "d", "ʒ", "y", " ", "ˌ", "u", "m", "ˈ", "ɐ", "̃", "n", "æ", " ", "d", "ʒ", "y", " ", "k", "l", "ˌ", "a", "s", "i", "f", "ˌ", "i", "k", "a", "s", "ˈ", "ɐ", "̃", "ʊ", "̃", " ", "d", "ʊ", "s", " ", "f", "ˌ", "e", "n", "ˈ", "o", "m", "e", "n", "ʊ", "z", " ", "d", "a", " ", "n", "ˌ", "a", "t", "u", "ɾ", "ˈ", "e", "z", "æ", ",", " ", "a", " ", "k", "ˌ", "a", "p", "a", "s", "i", "d", "ˈ", "a", "d", "ʒ", "y", " ", "f", "ˌ", "i", "n", "ˈ", "i", "t", "æ", " ", "d", "ʒ", "y", " ", "d", "ʒ", "ˌ", "i", "s", "t", "ʃ", "i", "ŋ", "s", "ˈ", "ɐ", "̃", "ʊ", "̃", " ", "d", "ʒ", "y", " ", "k", "ˈ", "o", "ɾ", "y", "s", " ", "p", "ˈ", "e", "l", "æ", " ", "v", "i", "z", "ˈ", "ɐ", "̃", "ʊ", "̃", " ", "ˌ", "u", "m", "ˈ", "ɐ", "̃", "n", "æ", " ", "i", " ", "p", "o", "r", " ", "k", "e", "s", "t", "ˈ", "o", "̃", "j", "z", " ", "d", "ʒ", "ˌ", "i", "d", "ˈ", "a", "t", "ʃ", "i", "k", "æ", "s", ",", " ", "u", " ", "ˈ", "a", "ɾ", "ə", "k", "ʊ", "ˈ", "i", "ɾ", "i", "z", " ", "ɛ", " ", "m", "ˈ", "a", "ɪ", "s", " ", "k", "ˌ", "o", "ɲ", "e", "s", "ˈ", "i", "d", "ʊ", " ", "p", "o", "ɾ", " ", "ˌ", "u", "m", "æ", " ", "s", "ˌ", "i", "m", "p", "l", "i", "f", "ˌ", "i", "k", "a", "s", "ˈ", "ɐ", "̃", "ʊ", "̃", " ", "k", "r", "ˌ", "i", "ˈ", "a", "d", "æ", " ", "k", "ˌ", "u", "w", "t", "u", "ɾ", "a", "ʊ", "m", "ˈ", "e", "ɪ", "ŋ", "t", "ʃ", "y", " ", "k", "y", " ", "x", "ˌ", "e", "z", "ˈ", "u", "m", "j", " ", "u", " ", "ˌ", "e", "s", "p", "ˈ", "ɛ", "k", "t", "r", "w", " ", "ˈ", "e", "ɪ", "ŋ", " ", "s", "ˈ", "ɛ", "t", "ʃ", "y", " ", "k", "ˈ", "o", "ɾ", "y", "z", " ", "n", "a", " ", "s", "ˌ", "e", "ɡ", "ˈ", "i", "ŋ", "t", "ʃ", "j", " ", "ˈ", "ɔ", "ɾ", "ə", "d", "e", "ɪ", "ŋ", ":", " ", "v", "ˌ", "e", "ɾ", "ə", "m", "ˈ", "e", "l", "j", "ʊ", ",", " ", "l", "ˌ", "a", "ɾ", "ˈ", "ɐ", "̃", "ŋ", "ʒ", "æ", ",", " ", "ˌ", "æ", "m", "a", "ɾ", "ˈ", "ɛ", "l", "ʊ", ",", " ", "v", "ˈ", "e", "ɾ", "ə", "d", "ʒ", "y", ",", " ", "a", "z", "ˈ", "u", "w", ",", " ", "ɐ", "̃", "n", "ˈ", "i", "ʊ", " ", "i", " ", "v", "ˌ", "i", "o", "l", "ˈ", "e", "t", "æ", "."], "phoneme_ids": [1, 0, 17, 0, 121, 0, 18, 0, 34, 0, 120, 0, 21, 0, 17, 0, 35, 0, 3, 0, 121, 0, 14, 0, 122, 0, 3, 0, 26, 0, 121, 0, 18, 0, 31, 0, 18, 0, 31, 0, 21, 0, 17, 0, 120, 0, 14, 0, 17, 0, 108, 0, 37, 0, 3, 0, 121, 0, 33, 0, 25, 0, 120, 0, 50, 0, 141, 0, 26, 0, 39, 0, 3, 0, 17, 0, 108, 0, 37, 0, 3, 0, 23, 0, 24, 0, 121, 0, 14, 0, 31, 0, 21, 0, 19, 0, 121, 0, 21, 0, 23, 0, 14, 0, 31, 0, 120, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 17, 0, 100, 0, 31, 0, 3, 0, 19, 0, 121, 0, 18, 0, 26, 0, 120, 0, 27, 0, 25, 0, 18, 0, 26, 0, 100, 0, 38, 0, 3, 0, 17, 0, 14, 0, 3, 0, 26, 0, 121, 0, 14, 0, 32, 0, 33, 0, 92, 0, 120, 0, 18, 0, 38, 0, 39, 0, 8, 0, 3, 0, 14, 0, 3, 0, 23, 0, 121, 0, 14, 0, 28, 0, 14, 0, 31, 0, 21, 0, 17, 0, 120, 0, 14, 0, 17, 0, 108, 0, 37, 0, 3, 0, 19, 0, 121, 0, 21, 0, 26, 0, 120, 0, 21, 0, 32, 0, 39, 0, 3, 0, 17, 0, 108, 0, 37, 0, 3, 0, 17, 0, 108, 0, 121, 0, 21, 0, 31, 0, 32, 0, 96, 0, 21, 0, 44, 0, 31, 0, 120, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 17, 0, 108, 0, 37, 0, 3, 0, 23, 0, 120, 0, 27, 0, 92, 0, 37, 0, 31, 0, 3, 0, 28, 0, 120, 0, 18, 0, 24, 0, 39, 0, 3, 0, 34, 0, 21, 0, 38, 0, 120, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 121, 0, 33, 0, 25, 0, 120, 0, 50, 0, 141, 0, 26, 0, 39, 0, 3, 0, 21, 0, 3, 0, 28, 0, 27, 0, 30, 0, 3, 0, 23, 0, 18, 0, 31, 0, 32, 0, 120, 0, 27, 0, 141, 0, 22, 0, 38, 0, 3, 0, 17, 0, 108, 0, 121, 0, 21, 0, 17, 0, 120, 0, 14, 0, 32, 0, 96, 0, 21, 0, 23, 0, 39, 0, 31, 0, 8, 0, 3, 0, 33, 0, 3, 0, 120, 0, 14, 0, 92, 0, 59, 0, 23, 0, 100, 0, 120, 0, 21, 0, 92, 0, 21, 0, 38, 0, 3, 0, 61, 0, 3, 0, 25, 0, 120, 0, 14, 0, 74, 0, 31, 0, 3, 0, 23, 0, 121, 0, 27, 0, 82, 0, 18, 0, 31, 0, 120, 0, 21, 0, 17, 0, 100, 0, 3, 0, 28, 0, 27, 0, 92, 0, 3, 0, 121, 0, 33, 0, 25, 0, 39, 0, 3, 0, 31, 0, 121, 0, 21, 0, 25, 0, 28, 0, 24, 0, 21, 0, 19, 0, 121, 0, 21, 0, 23, 0, 14, 0, 31, 0, 120, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 23, 0, 30, 0, 121, 0, 21, 0, 120, 0, 14, 0, 17, 0, 39, 0, 3, 0, 23, 0, 121, 0, 33, 0, 35, 0, 32, 0, 33, 0, 92, 0, 14, 0, 100, 0, 25, 0, 120, 0, 18, 0, 74, 0, 44, 0, 32, 0, 96, 0, 37, 0, 3, 0, 23, 0, 37, 0, 3, 0, 36, 0, 121, 0, 18, 0, 38, 0, 120, 0, 33, 0, 25, 0, 22, 0, 3, 0, 33, 0, 3, 0, 121, 0, 18, 0, 31, 0, 28, 0, 120, 0, 61, 0, 23, 0, 32, 0, 30, 0, 35, 0, 3, 0, 120, 0, 18, 0, 74, 0, 44, 0, 3, 0, 31, 0, 120, 0, 61, 0, 32, 0, 96, 0, 37, 0, 3, 0, 23, 0, 120, 0, 27, 0, 92, 0, 37, 0, 38, 0, 3, 0, 26, 0, 14, 0, 3, 0, 31, 0, 121, 0, 18, 0, 66, 0, 120, 0, 21, 0, 44, 0, 32, 0, 96, 0, 22, 0, 3, 0, 120, 0, 54, 0, 92, 0, 59, 0, 17, 0, 18, 0, 74, 0, 44, 0, 11, 0, 3, 0, 34, 0, 121, 0, 18, 0, 92, 0, 59, 0, 25, 0, 120, 0, 18, 0, 24, 0, 22, 0, 100, 0, 8, 0, 3, 0, 24, 0, 121, 0, 14, 0, 92, 0, 120, 0, 50, 0, 141, 0, 44, 0, 108, 0, 39, 0, 8, 0, 3, 0, 121, 0, 39, 0, 25, 0, 14, 0, 92, 0, 120, 0, 61, 0, 24, 0, 100, 0, 8, 0, 3, 0, 34, 0, 120, 0, 18, 0, 92, 0, 59, 0, 17, 0, 108, 0, 37, 0, 8, 0, 3, 0, 14, 0, 38, 0, 120, 0, 33, 0, 35, 0, 8, 0, 3, 0, 50, 0, 141, 0, 26, 0, 120, 0, 21, 0, 100, 0, 3, 0, 21, 0, 3, 0, 34, 0, 121, 0, 21, 0, 27, 0, 24, 0, 120, 0, 18, 0, 32, 0, 39, 0, 10, 0, 2]} -{"text": "Tal simplificação foi proposta primeiramente por Isaac Newton, que decidiu nomear apenas cinco cores e depois adicionou mais duas apenas para fazer analogia com as sete notas musicais, os sete dias da semana e os sete objetos do sistema solar conhecidos à época.", "phonemes": ["t", "ˈ", "a", "ʊ", " ", "s", "ˌ", "i", "m", "p", "l", "i", "f", "ˌ", "i", "k", "a", "s", "ˈ", "ɐ", "̃", "ʊ", "̃", " ", "f", "o", "ɪ", " ", "p", "r", "ˌ", "o", "p", "ˈ", "ɔ", "s", "t", "æ", " ", "p", "r", "ˌ", "i", "m", "e", "ɪ", "ɾ", "æ", "m", "ˈ", "e", "ɪ", "ŋ", "t", "ʃ", "y", " ", "p", "o", "ɾ", " ", "i", "z", "ˈ", "a", "c", " ", "n", "ˈ", "ɪ", "u", "t", "o", "ŋ", ",", " ", "k", "y", " ", "d", "ˌ", "e", "s", "i", "d", "ʒ", "ˈ", "i", "ʊ", " ", "n", "ˌ", "o", "m", "e", "ˈ", "a", "ɾ", " ", "ˌ", "a", "p", "ˈ", "e", "n", "æ", "s", " ", "s", "ˈ", "i", "ŋ", "k", "ʊ", " ", "k", "ˈ", "o", "ɾ", "y", "z", " ", "i", " ", "d", "e", "p", "ˈ", "o", "ɪ", "z", " ", "ˌ", "a", "d", "ʒ", "i", "s", "ˌ", "i", "o", "n", "ˈ", "o", "w", " ", "m", "ˈ", "a", "ɪ", "z", " ", "d", "ˈ", "u", "æ", "z", " ", "ˌ", "a", "p", "ˈ", "e", "n", "æ", "s", " ", "p", "ˌ", "a", "ɾ", "æ", " ", "f", "a", "z", "ˌ", "e", "ɾ", " ", "ˌ", "æ", "n", "a", "l", "o", "ʒ", "ˈ", "i", "æ", " ", "k", "o", "ŋ", " ", "a", "s", " ", "s", "ˈ", "ɛ", "t", "ʃ", "y", " ", "n", "ˈ", "ɔ", "t", "æ", "z", " ", "m", "ˌ", "u", "z", "i", "k", "ˈ", "a", "ɪ", "s", ",", " ", "ʊ", "s", " ", "s", "ˈ", "ɛ", "t", "ʃ", "y", " ", "d", "ʒ", "ˈ", "i", "æ", "z", " ", "d", "a", " ", "s", "ˌ", "e", "m", "ˈ", "ɐ", "̃", "n", "æ", " ", "i", " ", "ʊ", "s", " ", "s", "ˈ", "ɛ", "t", "ʃ", "j", " ", "ˌ", "o", "b", "ʒ", "ˈ", "ɛ", "t", "ʊ", "z", " ", "d", "ʊ", " ", "s", "ˌ", "i", "s", "t", "ˈ", "e", "m", "æ", " ", "s", "o", "l", "ˈ", "a", "r", " ", "k", "ˌ", "o", "ɲ", "e", "s", "ˈ", "i", "d", "ʊ", "z", " ", "ˌ", "a", "ː", " ", "ˈ", "ɛ", "p", "o", "k", "æ", "."], "phoneme_ids": [1, 0, 32, 0, 120, 0, 14, 0, 100, 0, 3, 0, 31, 0, 121, 0, 21, 0, 25, 0, 28, 0, 24, 0, 21, 0, 19, 0, 121, 0, 21, 0, 23, 0, 14, 0, 31, 0, 120, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 19, 0, 27, 0, 74, 0, 3, 0, 28, 0, 30, 0, 121, 0, 27, 0, 28, 0, 120, 0, 54, 0, 31, 0, 32, 0, 39, 0, 3, 0, 28, 0, 30, 0, 121, 0, 21, 0, 25, 0, 18, 0, 74, 0, 92, 0, 39, 0, 25, 0, 120, 0, 18, 0, 74, 0, 44, 0, 32, 0, 96, 0, 37, 0, 3, 0, 28, 0, 27, 0, 92, 0, 3, 0, 21, 0, 38, 0, 120, 0, 14, 0, 16, 0, 3, 0, 26, 0, 120, 0, 74, 0, 33, 0, 32, 0, 27, 0, 44, 0, 8, 0, 3, 0, 23, 0, 37, 0, 3, 0, 17, 0, 121, 0, 18, 0, 31, 0, 21, 0, 17, 0, 108, 0, 120, 0, 21, 0, 100, 0, 3, 0, 26, 0, 121, 0, 27, 0, 25, 0, 18, 0, 120, 0, 14, 0, 92, 0, 3, 0, 121, 0, 14, 0, 28, 0, 120, 0, 18, 0, 26, 0, 39, 0, 31, 0, 3, 0, 31, 0, 120, 0, 21, 0, 44, 0, 23, 0, 100, 0, 3, 0, 23, 0, 120, 0, 27, 0, 92, 0, 37, 0, 38, 0, 3, 0, 21, 0, 3, 0, 17, 0, 18, 0, 28, 0, 120, 0, 27, 0, 74, 0, 38, 0, 3, 0, 121, 0, 14, 0, 17, 0, 108, 0, 21, 0, 31, 0, 121, 0, 21, 0, 27, 0, 26, 0, 120, 0, 27, 0, 35, 0, 3, 0, 25, 0, 120, 0, 14, 0, 74, 0, 38, 0, 3, 0, 17, 0, 120, 0, 33, 0, 39, 0, 38, 0, 3, 0, 121, 0, 14, 0, 28, 0, 120, 0, 18, 0, 26, 0, 39, 0, 31, 0, 3, 0, 28, 0, 121, 0, 14, 0, 92, 0, 39, 0, 3, 0, 19, 0, 14, 0, 38, 0, 121, 0, 18, 0, 92, 0, 3, 0, 121, 0, 39, 0, 26, 0, 14, 0, 24, 0, 27, 0, 108, 0, 120, 0, 21, 0, 39, 0, 3, 0, 23, 0, 27, 0, 44, 0, 3, 0, 14, 0, 31, 0, 3, 0, 31, 0, 120, 0, 61, 0, 32, 0, 96, 0, 37, 0, 3, 0, 26, 0, 120, 0, 54, 0, 32, 0, 39, 0, 38, 0, 3, 0, 25, 0, 121, 0, 33, 0, 38, 0, 21, 0, 23, 0, 120, 0, 14, 0, 74, 0, 31, 0, 8, 0, 3, 0, 100, 0, 31, 0, 3, 0, 31, 0, 120, 0, 61, 0, 32, 0, 96, 0, 37, 0, 3, 0, 17, 0, 108, 0, 120, 0, 21, 0, 39, 0, 38, 0, 3, 0, 17, 0, 14, 0, 3, 0, 31, 0, 121, 0, 18, 0, 25, 0, 120, 0, 50, 0, 141, 0, 26, 0, 39, 0, 3, 0, 21, 0, 3, 0, 100, 0, 31, 0, 3, 0, 31, 0, 120, 0, 61, 0, 32, 0, 96, 0, 22, 0, 3, 0, 121, 0, 27, 0, 15, 0, 108, 0, 120, 0, 61, 0, 32, 0, 100, 0, 38, 0, 3, 0, 17, 0, 100, 0, 3, 0, 31, 0, 121, 0, 21, 0, 31, 0, 32, 0, 120, 0, 18, 0, 25, 0, 39, 0, 3, 0, 31, 0, 27, 0, 24, 0, 120, 0, 14, 0, 30, 0, 3, 0, 23, 0, 121, 0, 27, 0, 82, 0, 18, 0, 31, 0, 120, 0, 21, 0, 17, 0, 100, 0, 38, 0, 3, 0, 121, 0, 14, 0, 122, 0, 3, 0, 120, 0, 61, 0, 28, 0, 27, 0, 23, 0, 39, 0, 10, 0, 2]} +{"text": "Tal simplificação foi proposta primeiramente por Isaac Newton, que decidiu nomear apenas cinco cores e depois adicionou mais duas apenas para fazer analogia com as sete notas musicais, os sete dias da semana e os sete objetos do sistema solar conhecidos à época.", "phonemes": ["t", "ˈ", "a", "ʊ", " ", "s", "ˌ", "i", "m", "p", "l", "i", "f", "ˌ", "i", "k", "a", "s", "ˈ", "ɐ", "̃", "ʊ", "̃", " ", "f", "o", "ɪ", " ", "p", "r", "ˌ", "o", "p", "ˈ", "ɔ", "s", "t", "æ", " ", "p", "r", "ˌ", "i", "m", "e", "ɪ", "ɾ", "æ", "m", "ˈ", "e", "ɪ", "ŋ", "t", "ʃ", "y", " ", "p", "o", "ɾ", " ", "i", "z", "ˈ", "a", "k", " ", "n", "ˈ", "ɪ", "u", "t", "o", "ŋ", ",", " ", "k", "y", " ", "d", "ˌ", "e", "s", "i", "d", "ʒ", "ˈ", "i", "ʊ", " ", "n", "ˌ", "o", "m", "e", "ˈ", "a", "ɾ", " ", "ˌ", "a", "p", "ˈ", "e", "n", "æ", "s", " ", "s", "ˈ", "i", "ŋ", "k", "ʊ", " ", "k", "ˈ", "o", "ɾ", "y", "z", " ", "i", " ", "d", "e", "p", "ˈ", "o", "ɪ", "z", " ", "ˌ", "a", "d", "ʒ", "i", "s", "ˌ", "i", "o", "n", "ˈ", "o", "w", " ", "m", "ˈ", "a", "ɪ", "z", " ", "d", "ˈ", "u", "æ", "z", " ", "ˌ", "a", "p", "ˈ", "e", "n", "æ", "s", " ", "p", "ˌ", "a", "ɾ", "æ", " ", "f", "a", "z", "ˌ", "e", "ɾ", " ", "ˌ", "æ", "n", "a", "l", "o", "ʒ", "ˈ", "i", "æ", " ", "k", "o", "ŋ", " ", "a", "s", " ", "s", "ˈ", "ɛ", "t", "ʃ", "y", " ", "n", "ˈ", "ɔ", "t", "æ", "z", " ", "m", "ˌ", "u", "z", "i", "k", "ˈ", "a", "ɪ", "s", ",", " ", "ʊ", "s", " ", "s", "ˈ", "ɛ", "t", "ʃ", "y", " ", "d", "ʒ", "ˈ", "i", "æ", "z", " ", "d", "a", " ", "s", "ˌ", "e", "m", "ˈ", "ɐ", "̃", "n", "æ", " ", "i", " ", "ʊ", "s", " ", "s", "ˈ", "ɛ", "t", "ʃ", "j", " ", "ˌ", "o", "b", "ʒ", "ˈ", "ɛ", "t", "ʊ", "z", " ", "d", "ʊ", " ", "s", "ˌ", "i", "s", "t", "ˈ", "e", "m", "æ", " ", "s", "o", "l", "ˈ", "a", "r", " ", "k", "ˌ", "o", "ɲ", "e", "s", "ˈ", "i", "d", "ʊ", "z", " ", "ˌ", "a", "ː", " ", "ˈ", "ɛ", "p", "o", "k", "æ", "."], "phoneme_ids": [1, 0, 32, 0, 120, 0, 14, 0, 100, 0, 3, 0, 31, 0, 121, 0, 21, 0, 25, 0, 28, 0, 24, 0, 21, 0, 19, 0, 121, 0, 21, 0, 23, 0, 14, 0, 31, 0, 120, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 19, 0, 27, 0, 74, 0, 3, 0, 28, 0, 30, 0, 121, 0, 27, 0, 28, 0, 120, 0, 54, 0, 31, 0, 32, 0, 39, 0, 3, 0, 28, 0, 30, 0, 121, 0, 21, 0, 25, 0, 18, 0, 74, 0, 92, 0, 39, 0, 25, 0, 120, 0, 18, 0, 74, 0, 44, 0, 32, 0, 96, 0, 37, 0, 3, 0, 28, 0, 27, 0, 92, 0, 3, 0, 21, 0, 38, 0, 120, 0, 14, 0, 23, 0, 3, 0, 26, 0, 120, 0, 74, 0, 33, 0, 32, 0, 27, 0, 44, 0, 8, 0, 3, 0, 23, 0, 37, 0, 3, 0, 17, 0, 121, 0, 18, 0, 31, 0, 21, 0, 17, 0, 108, 0, 120, 0, 21, 0, 100, 0, 3, 0, 26, 0, 121, 0, 27, 0, 25, 0, 18, 0, 120, 0, 14, 0, 92, 0, 3, 0, 121, 0, 14, 0, 28, 0, 120, 0, 18, 0, 26, 0, 39, 0, 31, 0, 3, 0, 31, 0, 120, 0, 21, 0, 44, 0, 23, 0, 100, 0, 3, 0, 23, 0, 120, 0, 27, 0, 92, 0, 37, 0, 38, 0, 3, 0, 21, 0, 3, 0, 17, 0, 18, 0, 28, 0, 120, 0, 27, 0, 74, 0, 38, 0, 3, 0, 121, 0, 14, 0, 17, 0, 108, 0, 21, 0, 31, 0, 121, 0, 21, 0, 27, 0, 26, 0, 120, 0, 27, 0, 35, 0, 3, 0, 25, 0, 120, 0, 14, 0, 74, 0, 38, 0, 3, 0, 17, 0, 120, 0, 33, 0, 39, 0, 38, 0, 3, 0, 121, 0, 14, 0, 28, 0, 120, 0, 18, 0, 26, 0, 39, 0, 31, 0, 3, 0, 28, 0, 121, 0, 14, 0, 92, 0, 39, 0, 3, 0, 19, 0, 14, 0, 38, 0, 121, 0, 18, 0, 92, 0, 3, 0, 121, 0, 39, 0, 26, 0, 14, 0, 24, 0, 27, 0, 108, 0, 120, 0, 21, 0, 39, 0, 3, 0, 23, 0, 27, 0, 44, 0, 3, 0, 14, 0, 31, 0, 3, 0, 31, 0, 120, 0, 61, 0, 32, 0, 96, 0, 37, 0, 3, 0, 26, 0, 120, 0, 54, 0, 32, 0, 39, 0, 38, 0, 3, 0, 25, 0, 121, 0, 33, 0, 38, 0, 21, 0, 23, 0, 120, 0, 14, 0, 74, 0, 31, 0, 8, 0, 3, 0, 100, 0, 31, 0, 3, 0, 31, 0, 120, 0, 61, 0, 32, 0, 96, 0, 37, 0, 3, 0, 17, 0, 108, 0, 120, 0, 21, 0, 39, 0, 38, 0, 3, 0, 17, 0, 14, 0, 3, 0, 31, 0, 121, 0, 18, 0, 25, 0, 120, 0, 50, 0, 141, 0, 26, 0, 39, 0, 3, 0, 21, 0, 3, 0, 100, 0, 31, 0, 3, 0, 31, 0, 120, 0, 61, 0, 32, 0, 96, 0, 22, 0, 3, 0, 121, 0, 27, 0, 15, 0, 108, 0, 120, 0, 61, 0, 32, 0, 100, 0, 38, 0, 3, 0, 17, 0, 100, 0, 3, 0, 31, 0, 121, 0, 21, 0, 31, 0, 32, 0, 120, 0, 18, 0, 25, 0, 39, 0, 3, 0, 31, 0, 27, 0, 24, 0, 120, 0, 14, 0, 30, 0, 3, 0, 23, 0, 121, 0, 27, 0, 82, 0, 18, 0, 31, 0, 120, 0, 21, 0, 17, 0, 100, 0, 38, 0, 3, 0, 121, 0, 14, 0, 122, 0, 3, 0, 120, 0, 61, 0, 28, 0, 27, 0, 23, 0, 39, 0, 10, 0, 2]} {"text": "Para informações sobre o espectro de cores do arco-íris, veja também o artigo sobre cores.", "phonemes": ["p", "ˌ", "a", "ɾ", "æ", " ", "ˌ", "i", "ŋ", "f", "o", "ɾ", "ə", "m", "a", "s", "ˈ", "o", "̃", "j", "s", " ", "s", "ˈ", "o", "b", "r", "i", " ", "u", " ", "ˌ", "e", "s", "p", "ˈ", "ɛ", "k", "t", "r", "ʊ", " ", "d", "ʒ", "y", " ", "k", "ˈ", "o", "ɾ", "y", "z", " ", "d", "ʊ", " ", "ˈ", "a", "ɾ", "ə", "k", "ʊ", "ˈ", "i", "ɾ", "i", "s", ",", " ", "v", "ˈ", "e", "ʒ", "æ", " ", "t", "ɐ", "̃", "m", "b", "ˈ", "e", "ɪ", "ŋ", " ", "u", " ", "ˌ", "a", "ɾ", "ə", "t", "ʃ", "ˈ", "i", "ɡ", "ʊ", " ", "s", "ˈ", "o", "b", "r", "y", " ", "k", "ˈ", "o", "ɾ", "y", "s", "."], "phoneme_ids": [1, 0, 28, 0, 121, 0, 14, 0, 92, 0, 39, 0, 3, 0, 121, 0, 21, 0, 44, 0, 19, 0, 27, 0, 92, 0, 59, 0, 25, 0, 14, 0, 31, 0, 120, 0, 27, 0, 141, 0, 22, 0, 31, 0, 3, 0, 31, 0, 120, 0, 27, 0, 15, 0, 30, 0, 21, 0, 3, 0, 33, 0, 3, 0, 121, 0, 18, 0, 31, 0, 28, 0, 120, 0, 61, 0, 23, 0, 32, 0, 30, 0, 100, 0, 3, 0, 17, 0, 108, 0, 37, 0, 3, 0, 23, 0, 120, 0, 27, 0, 92, 0, 37, 0, 38, 0, 3, 0, 17, 0, 100, 0, 3, 0, 120, 0, 14, 0, 92, 0, 59, 0, 23, 0, 100, 0, 120, 0, 21, 0, 92, 0, 21, 0, 31, 0, 8, 0, 3, 0, 34, 0, 120, 0, 18, 0, 108, 0, 39, 0, 3, 0, 32, 0, 50, 0, 141, 0, 25, 0, 15, 0, 120, 0, 18, 0, 74, 0, 44, 0, 3, 0, 33, 0, 3, 0, 121, 0, 14, 0, 92, 0, 59, 0, 32, 0, 96, 0, 120, 0, 21, 0, 66, 0, 100, 0, 3, 0, 31, 0, 120, 0, 27, 0, 15, 0, 30, 0, 37, 0, 3, 0, 23, 0, 120, 0, 27, 0, 92, 0, 37, 0, 31, 0, 10, 0, 2]} {"text": "Luís argüia à Júlia que «brações, fé, chá, óxido, pôr, zângão» eram palavras do português.", "phonemes": ["l", "w", "ˈ", "i", "z", " ", "ˌ", "a", "ɾ", "ə", "ɡ", "u", "ˈ", "i", " ", "ˌ", "a", "ː", " ", "ʒ", "ˈ", "u", "l", "j", "æ", " ", "k", "y", " ", "b", "r", "a", "s", "ˈ", "o", "̃", "j", "s", ",", " ", "f", "ˈ", "ɛ", ",", " ", "ʃ", "ˈ", "a", ",", " ", "ˈ", "ɔ", "k", "s", "i", "d", "ʊ", ",", " ", "p", "ˈ", "o", "r", ",", " ", "z", "ˈ", "ɐ", "̃", "ŋ", "ɡ", "ɐ", "̃", "ʊ", "̃", " ", "ˌ", "ɛ", "ɾ", "ɐ", "̃", "ʊ", "̃", " ", "p", "ˌ", "a", "l", "ˈ", "a", "v", "r", "æ", "z", " ", "d", "ʊ", " ", "p", "ˌ", "o", "ɾ", "ə", "t", "u", "ɡ", "ˈ", "e", "s", "."], "phoneme_ids": [1, 0, 24, 0, 35, 0, 120, 0, 21, 0, 38, 0, 3, 0, 121, 0, 14, 0, 92, 0, 59, 0, 66, 0, 33, 0, 120, 0, 21, 0, 3, 0, 121, 0, 14, 0, 122, 0, 3, 0, 108, 0, 120, 0, 33, 0, 24, 0, 22, 0, 39, 0, 3, 0, 23, 0, 37, 0, 3, 0, 15, 0, 30, 0, 14, 0, 31, 0, 120, 0, 27, 0, 141, 0, 22, 0, 31, 0, 8, 0, 3, 0, 19, 0, 120, 0, 61, 0, 8, 0, 3, 0, 96, 0, 120, 0, 14, 0, 8, 0, 3, 0, 120, 0, 54, 0, 23, 0, 31, 0, 21, 0, 17, 0, 100, 0, 8, 0, 3, 0, 28, 0, 120, 0, 27, 0, 30, 0, 8, 0, 3, 0, 38, 0, 120, 0, 50, 0, 141, 0, 44, 0, 66, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 121, 0, 61, 0, 92, 0, 50, 0, 141, 0, 100, 0, 141, 0, 3, 0, 28, 0, 121, 0, 14, 0, 24, 0, 120, 0, 14, 0, 34, 0, 30, 0, 39, 0, 38, 0, 3, 0, 17, 0, 100, 0, 3, 0, 28, 0, 121, 0, 27, 0, 92, 0, 59, 0, 32, 0, 33, 0, 66, 0, 120, 0, 18, 0, 31, 0, 10, 0, 2]} {"text": "À noite, vovô Kowalsky vê o ímã cair no pé do pingüim queixoso e vovó põe açúcar no chá de tâmaras do jabuti feliz.", "phonemes": ["ˌ", "a", "ː", " ", "n", "ˈ", "o", "ɪ", "t", "ʃ", "y", ",", " ", "v", "o", "v", "ˈ", "o", " ", "k", "ˌ", "o", "w", "ˈ", "a", "ʊ", "s", "k", "i", " ", "v", "ˈ", "e", " ", "u", " ", "ˈ", "i", "m", "ɐ", "̃", " ", "k", "a", "ˈ", "i", "r", " ", "n", "ʊ", " ", "p", "ˈ", "ɛ", " ", "d", "ʊ", " ", "p", "ˌ", "i", "ŋ", "ɡ", "u", "ˈ", "i", "ŋ", " ", "k", "ˌ", "e", "ɪ", "ʃ", "ˈ", "o", "z", "w", " ", "i", " ", "v", "o", "v", "ˈ", "ɔ", " ", "p", "ˈ", "o", "̃", "j", " ", "ˌ", "a", "s", "ˈ", "u", "k", "a", "r", " ", "n", "ʊ", " ", "ʃ", "ˈ", "a", " ", "d", "ʒ", "y", " ", "t", "ˈ", "ɐ", "̃", "m", "æ", "ɾ", "æ", "z", " ", "d", "ʊ", " ", "ʒ", "ˌ", "a", "b", "u", "t", "ʃ", "ˈ", "i", " ", "f", "e", "l", "ˈ", "i", "s", "."], "phoneme_ids": [1, 0, 121, 0, 14, 0, 122, 0, 3, 0, 26, 0, 120, 0, 27, 0, 74, 0, 32, 0, 96, 0, 37, 0, 8, 0, 3, 0, 34, 0, 27, 0, 34, 0, 120, 0, 27, 0, 3, 0, 23, 0, 121, 0, 27, 0, 35, 0, 120, 0, 14, 0, 100, 0, 31, 0, 23, 0, 21, 0, 3, 0, 34, 0, 120, 0, 18, 0, 3, 0, 33, 0, 3, 0, 120, 0, 21, 0, 25, 0, 50, 0, 141, 0, 3, 0, 23, 0, 14, 0, 120, 0, 21, 0, 30, 0, 3, 0, 26, 0, 100, 0, 3, 0, 28, 0, 120, 0, 61, 0, 3, 0, 17, 0, 100, 0, 3, 0, 28, 0, 121, 0, 21, 0, 44, 0, 66, 0, 33, 0, 120, 0, 21, 0, 44, 0, 3, 0, 23, 0, 121, 0, 18, 0, 74, 0, 96, 0, 120, 0, 27, 0, 38, 0, 35, 0, 3, 0, 21, 0, 3, 0, 34, 0, 27, 0, 34, 0, 120, 0, 54, 0, 3, 0, 28, 0, 120, 0, 27, 0, 141, 0, 22, 0, 3, 0, 121, 0, 14, 0, 31, 0, 120, 0, 33, 0, 23, 0, 14, 0, 30, 0, 3, 0, 26, 0, 100, 0, 3, 0, 96, 0, 120, 0, 14, 0, 3, 0, 17, 0, 108, 0, 37, 0, 3, 0, 32, 0, 120, 0, 50, 0, 141, 0, 25, 0, 39, 0, 92, 0, 39, 0, 38, 0, 3, 0, 17, 0, 100, 0, 3, 0, 108, 0, 121, 0, 14, 0, 15, 0, 33, 0, 32, 0, 96, 0, 120, 0, 21, 0, 3, 0, 19, 0, 18, 0, 24, 0, 120, 0, 21, 0, 31, 0, 10, 0, 2]} diff --git a/src/python/piper_train/phonemize.py b/src/python/piper_train/phonemize.py index a5065bf..e1f4243 100644 --- a/src/python/piper_train/phonemize.py +++ b/src/python/piper_train/phonemize.py @@ -173,6 +173,11 @@ DEFAULT_PHONEME_ID_MAP: Dict[str, List[int]] = { "↑": [151], } +PHONEME_MAPS = { + # Brazilian Portuguese + "pt-br": {"c": ["k"]} +} + ALPHABETS = { # Ukrainian "uk": { @@ -224,16 +229,33 @@ ALPHABETS = { "\u0301": [45], # combining acute accent "\u0306": [46], # combining breve "\u0308": [47], # combining diaeresis - "—": [48], # em dash + "—": [48], # em dash } } -def phonemize(text: str, phonemizer: Phonemizer) -> List[str]: +def phonemize( + text: str, + phonemizer: Phonemizer, + phoneme_map: Optional[Dict[str, List[str]]] = None, +) -> List[str]: phonemes_str = phonemizer.phonemize(text=text, keep_clause_breakers=True) # Phonemes are decomposed into unicode codepoints - return list(unicodedata.normalize("NFD", phonemes_str)) + unmapped_phonemes = list(unicodedata.normalize("NFD", phonemes_str)) + if not phoneme_map: + return unmapped_phonemes + + # Phonemes can be mapped to lists of other phonemes + mapped_phonemes = [] + for phoneme in unmapped_phonemes: + sub_phonemes = phoneme_map.get(phoneme) + if sub_phonemes: + mapped_phonemes.extend(sub_phonemes) + else: + mapped_phonemes.append(phoneme) + + return mapped_phonemes def phonemes_to_ids( @@ -281,13 +303,14 @@ def main() -> None: args = parser.parse_args() phonemizer = Phonemizer(args.language) + phoneme_map = PHONEME_MAPS.get(args.language) for line in sys.stdin: line = line.strip() if not line: continue - phonemes = phonemize(line, phonemizer) + phonemes = phonemize(line, phonemizer, phoneme_map=phoneme_map) phoneme_ids = phonemes_to_ids(phonemes) json.dump( { diff --git a/src/python/piper_train/preprocess.py b/src/python/piper_train/preprocess.py index 73f71ac..8d081bb 100644 --- a/src/python/piper_train/preprocess.py +++ b/src/python/piper_train/preprocess.py @@ -24,6 +24,7 @@ from .phonemize import ( PhonemeType, phonemes_to_ids, phonemize, + PHONEME_MAPS, ) _LOGGER = logging.getLogger("preprocess") @@ -250,6 +251,7 @@ def phonemize_batch_espeak( casing = get_text_casing(args.text_casing) silence_detector = make_silence_detector() phonemizer = Phonemizer(default_voice=args.language) + phoneme_map = PHONEME_MAPS.get(args.langauge) while True: utt_batch = queue_in.get() @@ -259,7 +261,9 @@ def phonemize_batch_espeak( for utt in utt_batch: try: _LOGGER.debug(utt) - utt.phonemes = phonemize(casing(utt.text), phonemizer) + utt.phonemes = phonemize( + casing(utt.text), phonemizer, phoneme_map=phoneme_map + ) utt.phoneme_ids = phonemes_to_ids( utt.phonemes, missing_phonemes=utt.missing_phonemes,