protected byte[] performEspeak(CommunicateAction communicateAction, Locale lang)
throws IOException {
byte[] wavBytes;
try (final ByteArrayInputStream objectIn =
new ByteArrayInputStream(
communicateAction.getObject().getBytes(StandardCharsets.UTF_8));
final ByteArrayOutputStream wavStream = new ByteArrayOutputStream();
final ByteArrayOutputStream err = new ByteArrayOutputStream()) {
final CommandLine cmdLine = new CommandLine("espeak");
cmdLine.addArgument("-b");
cmdLine.addArgument("1"); // UTF-8
cmdLine.addArgument("-m"); // SSML markup
cmdLine.addArgument("-s");
cmdLine.addArgument("130");
if (INDONESIAN.getLanguage().equals(lang.getLanguage())) {
cmdLine.addArgument("-v");
cmdLine.addArgument(SpeechProsody.MBROLA_ID1_VOICE);
cmdLine.addArgument("-a");
cmdLine.addArgument(String.valueOf(INDONESIAN_AMPLITUDE));
} else if ("ar".equals(lang.getLanguage())) {
cmdLine.addArgument("-v");
cmdLine.addArgument(SpeechProsody.MBROLA_AR1_VOICE);
}
// cmdLine.addArgument("-w");
// cmdLine.addArgument(wavFile.toString());
cmdLine.addArgument("--stdin");
cmdLine.addArgument("--stdout");
// cmdLine.addArgument(communicateAction.getObject());
executor.setStreamHandler(new PumpStreamHandler(wavStream, err, objectIn));
final int executed;
try {
executed = executor.execute(cmdLine);
wavBytes = wavStream.toByteArray();
} finally {
log.info("{}: {}", cmdLine, err.toString());
}
}
return wavBytes;
}
protected Status processCommunicateAction(
final Exchange exchange, final CommunicateAction communicateAction) throws IOException {
final EmotionKind emotionKind =
Optional.ofNullable(communicateAction.getEmotionKind()).orElse(EmotionKind.NEUTRAL);
final Locale lang = Optional.ofNullable(communicateAction.getInLanguage()).orElse(Locale.US);
log.info("Got speech lang-legacy={}: {}", lang.getLanguage(), communicateAction);
final String avatarId = Optional.ofNullable(communicateAction.getAvatarId()).orElse("nao1");
// final File wavFile = File.createTempFile("lumen-speech-synthesis_", ".wav");
// final File oggFile = File.createTempFile("lumen-speech-synthesis_",
// ".ogg");
try {
byte[] wavBytes = null;
if (INDONESIAN.getLanguage().equals(lang.getLanguage())) {
// Expressive speech (for now, Indonesian only)
try {
PhonemeDoc phonemeDoc;
if (EmotionKind.NEUTRAL == emotionKind) {
phonemeDoc = speechProsody.performNeutral(communicateAction.getObject());
} else {
try {
final EmotionProsody emotionProsody =
emotionProsodies
.getEmotion(emotionKind)
.orElseThrow(
() ->
new SpeechSynthesisException(
"Emotion " + emotionKind + " not supported"));
phonemeDoc = speechProsody.perform(communicateAction.getObject(), emotionProsody);
} catch (Exception e) {
log.error(
"Cannot speak with emotion "
+ emotionKind
+ ", falling back to NEUTRAL: "
+ communicateAction.getObject(),
e);
phonemeDoc = speechProsody.performNeutral(communicateAction.getObject());
}
}
try (final ByteArrayInputStream objectIn =
new ByteArrayInputStream(phonemeDoc.toString().getBytes(StandardCharsets.UTF_8));
final ByteArrayOutputStream wavStream = new ByteArrayOutputStream();
final ByteArrayOutputStream err = new ByteArrayOutputStream()) {
final CommandLine cmdLine = new CommandLine("mbrola");
cmdLine.addArgument("-v");
cmdLine.addArgument(String.valueOf(INDONESIAN_AMPLITUDE / 100f));
cmdLine.addArgument(new File(mbrolaShareFolder, "id1/id1").toString());
cmdLine.addArgument("-");
cmdLine.addArgument("-.wav");
executor.setStreamHandler(new PumpStreamHandler(wavStream, err, objectIn));
final int executed;
try {
executed = executor.execute(cmdLine);
wavBytes = wavStream.toByteArray();
} finally {
log.info("{}: {}", cmdLine, err.toString());
}
}
} catch (Exception e) {
log.error(
"Cannot speak Indonesian using prosody engine, falling back to direct espeak: "
+ communicateAction.getObject(),
e);
}
}
if (wavBytes == null) {
// Neutral speech using direct espeak
try {
wavBytes = performEspeak(communicateAction, lang);
} catch (Exception e) {
if (!Locale.US.getLanguage().equals(lang.getLanguage())) {
// Indonesian sometimes fails especially "k-k", e.g. "baik koq".
// retry using English as last resort, as long as it says something!
log.error(
"Cannot speak using "
+ lang.toLanguageTag()
+ ", falling back to English (US): "
+ communicateAction.getObject(),
e);
wavBytes = performEspeak(communicateAction, Locale.US);
} else {
throw e;
}
}
}
log.info("espeak/mbrola generated {} bytes WAV", wavBytes.length);
try (final ByteArrayInputStream wavIn = new ByteArrayInputStream(wavBytes);
final ByteArrayOutputStream bos = new ByteArrayOutputStream();
final ByteArrayOutputStream err = new ByteArrayOutputStream()) {
// flac.exe doesn't support mp3, and that's a problem for now (note: mp3 patent is expiring)
final CommandLine cmdLine = new CommandLine(ffmpegExecutable);
cmdLine.addArgument("-i");
cmdLine.addArgument("-"); // cmdLine.addArgument(wavFile.toString());
cmdLine.addArgument("-ar");
cmdLine.addArgument(String.valueOf(SAMPLE_RATE));
cmdLine.addArgument("-ac");
cmdLine.addArgument("1");
cmdLine.addArgument("-f");
cmdLine.addArgument("ogg");
// without this you'll get FLAC instead, which browsers do not support
cmdLine.addArgument("-acodec");
cmdLine.addArgument("libvorbis");
// cmdLine.addArgument("-y"); // happens, weird!
// cmdLine.addArgument(oggFile.toString());
cmdLine.addArgument("-");
executor.setStreamHandler(new PumpStreamHandler(bos, err, wavIn));
final int executed;
try {
executed = executor.execute(cmdLine);
} finally {
log.info("{}: {}", cmdLine, err.toString());
}
// Preconditions.checkState(oggFile.exists(), "Cannot convert
// %s bytes WAV to OGG",
// wavBytes.length);
// Send
// final byte[] audioContent =
// FileUtils.readFileToByteArray(oggFile);
final byte[] audioContent = bos.toByteArray();
final String audioContentType = "audio/ogg";
final AudioObject audioObject = new AudioObject();
audioObject.setTranscript(communicateAction.getObject());
audioObject.setInLanguage(lang);
audioObject.setMediaLayer(MediaLayer.SPEECH);
audioObject.setContentType(audioContentType + "; rate=" + SAMPLE_RATE);
audioObject.setContentUrl(
"data:" + audioContentType + ";base64," + Base64.encodeBase64String(audioContent));
audioObject.setContentSize((long) audioContent.length);
//
// audioObject.setName(FilenameUtils.getName(oggFile.getName()));
audioObject.setName("lumen-speech-" + new DateTime() + ".ogg");
audioObject.setDateCreated(new DateTime());
audioObject.setDatePublished(audioObject.getDateCreated());
audioObject.setDateModified(audioObject.getDateCreated());
audioObject.setUploadDate(audioObject.getDateCreated());
final String audioOutUri =
"rabbitmq://dummy/amq.topic?connectionFactory=#amqpConnFactory&exchangeType=topic&autoDelete=false&skipQueueDeclare=true&routingKey="
+ AvatarChannel.AUDIO_OUT.key(avatarId);
log.info("Sending {} to {} ...", audioObject, audioOutUri);
producer.sendBodyAndHeader(
audioOutUri,
toJson.getMapper().writeValueAsBytes(audioObject),
RabbitMQConstants.EXPIRATION,
String.valueOf(MESSAGE_EXPIRATION.getMillis()));
}
} finally {
// oggFile.delete();
// wavFile.delete();
}
// reply
log.trace("Exchange {} is {}", exchange.getIn().getMessageId(), exchange.getPattern());
final Status status = new Status();
exchange.getOut().setBody(status);
return status;
// final String replyTo = exchange.getIn().getHeader("rabbitmq.REPLY_TO",
// String.class);
// if (replyTo != null) {
// log.debug("Sending reply to {} ...", replyTo);
// exchange.getOut().setHeader("rabbitmq.ROUTING_KEY", replyTo);
// exchange.getOut().setHeader("rabbitmq.EXCHANGE_NAME", "");
// exchange.getOut().setHeader("recipients",
//
// "rabbitmq://dummy/dummy?connectionFactory=#amqpConnFactory&autoDelete=false,log:OUT." +
// LumenChannel.SPEECH_SYNTHESIS);
// } else {
// exchange.getOut().setHeader("recipients");
// }
}
@Override
public void configure() throws Exception {
onException(Exception.class).bean(asError).bean(toJson).handled(true);
errorHandler(new LoggingErrorHandlerBuilder(log));
// lumen.speech.synthesis
from("rabbitmq://localhost/amq.topic?connectionFactory=#amqpConnFactory&exchangeType=topic&autoDelete=false&queue=speech-synthesis:"
+ LumenChannel.SPEECH_SYNTHESIS.key()
+ "&routingKey="
+ LumenChannel.SPEECH_SYNTHESIS.key())
.to(
"log:IN."
+ LumenChannel.SPEECH_SYNTHESIS.key()
+ "?showHeaders=true&showAll=true&multiline=true")
.process(
exchange -> {
final LumenThing thing =
toJson
.getMapper()
.readValue(exchange.getIn().getBody(byte[].class), LumenThing.class);
if (thing instanceof CommunicateAction) {
final CommunicateAction communicateAction = (CommunicateAction) thing;
processCommunicateAction(exchange, communicateAction);
} else {
// do not reply
exchange.getOut().setBody(null);
}
})
.bean(toJson);
// .to("log:OUT." + LumenChannel.SPEECH_SYNTHESIS);
// avatar.*.chat.outbox
from("rabbitmq://localhost/amq.topic?connectionFactory=#amqpConnFactory&exchangeType=topic&autoDelete=false&queue=speech-synthesis:"
+ AvatarChannel.CHAT_OUTBOX.wildcard()
+ "&routingKey="
+ AvatarChannel.CHAT_OUTBOX.wildcard())
.to(
"log:"
+ SpeechSynthesisRouter.class.getName()
+ "."
+ AvatarChannel.CHAT_OUTBOX.wildcard()
+ "?level=DEBUG&showHeaders=true&showBody=false&multiline=true")
.process(
exchange -> {
final String avatarId =
AvatarChannel.getAvatarId(
(String) exchange.getIn().getHeader(RabbitMQConstants.ROUTING_KEY));
final LumenThing thing =
toJson
.getMapper()
.readValue(exchange.getIn().getBody(byte[].class), LumenThing.class);
if (thing instanceof CommunicateAction) {
final CommunicateAction communicateAction = (CommunicateAction) thing;
if (Boolean.TRUE == communicateAction.getUsedForSynthesis()) {
processCommunicateAction(exchange, communicateAction);
} else {
// do not reply
exchange.getOut().setBody(null);
}
}
// we never reply anyway
exchange.setPattern(ExchangePattern.InOnly);
});
}