Add hacky --alignmentFile option

rhubarb/src/lib/rhubarbLib.cpp

@@ -11,13 +11,14 @@ using std::filesystem::path;
 JoiningContinuousTimeline<Shape> animateAudioClip(
 	const AudioClip& audioClip,
 	const optional<string>& dialog,
+	const optional<BoundedTimeline<Phone>>& alignedPhones,
 	const Recognizer& recognizer,
 	const ShapeSet& targetShapeSet,
 	int maxThreadCount,
 	ProgressSink& progressSink)
 {
 	const BoundedTimeline<Phone> phones =
-		recognizer.recognizePhones(audioClip, dialog, maxThreadCount, progressSink);
+		recognizer.recognizePhones(audioClip, dialog, alignedPhones, maxThreadCount, progressSink);
 	JoiningContinuousTimeline<Shape> result = animate(phones, targetShapeSet);
 	return result;
 }
@@ -25,11 +26,12 @@ JoiningContinuousTimeline<Shape> animateAudioClip(
 JoiningContinuousTimeline<Shape> animateWaveFile(
 	path filePath,
 	const optional<string>& dialog,
+	const optional<BoundedTimeline<Phone>>& alignedPhones,
 	const Recognizer& recognizer,
 	const ShapeSet& targetShapeSet,
 	int maxThreadCount,
 	ProgressSink& progressSink)
 {
 	const auto audioClip = createAudioFileClip(filePath);
-	return animateAudioClip(*audioClip, dialog, recognizer, targetShapeSet, maxThreadCount, progressSink);
+	return animateAudioClip(*audioClip, dialog, alignedPhones, recognizer, targetShapeSet, maxThreadCount, progressSink);
 }

rhubarb/src/lib/rhubarbLib.h

@@ -11,6 +11,7 @@
 JoiningContinuousTimeline<Shape> animateAudioClip(
 	const AudioClip& audioClip,
 	const boost::optional<std::string>& dialog,
+	const boost::optional<BoundedTimeline<Phone>>& alignedPhones,
 	const Recognizer& recognizer,
 	const ShapeSet& targetShapeSet,
 	int maxThreadCount,
@@ -19,6 +20,7 @@ JoiningContinuousTimeline<Shape> animateAudioClip(
 JoiningContinuousTimeline<Shape> animateWaveFile(
 	std::filesystem::path filePath,
 	const boost::optional<std::string>& dialog,
+	const boost::optional<BoundedTimeline<Phone>>& alignedPhones,
 	const Recognizer& recognizer,
 	const ShapeSet& targetShapeSet,
 	int maxThreadCount,

rhubarb/src/recognition/PhoneticRecognizer.cpp

@@ -106,8 +106,12 @@ static Timeline<Phone> utteranceToPhones(
 BoundedTimeline<Phone> PhoneticRecognizer::recognizePhones(
 	const AudioClip& inputAudioClip,
 	optional<std::string> dialog,
+	optional<BoundedTimeline<Phone>> alignedPhones,
 	int maxThreadCount,
 	ProgressSink& progressSink
 ) const {
+	if (alignedPhones) {
+		throw std::invalid_argument("Phonetic recognizer doesn't support specifying aligned phones in this POC.");
+	}
 	return ::recognizePhones(inputAudioClip, dialog, &createDecoder, &utteranceToPhones, maxThreadCount, progressSink);
 }

rhubarb/src/recognition/PhoneticRecognizer.h

@@ -8,6 +8,7 @@ public:
 	BoundedTimeline<Phone> recognizePhones(
 		const AudioClip& inputAudioClip,
 		boost::optional<std::string> dialog,
+		boost::optional<BoundedTimeline<Phone>> alignedPhones,
 		int maxThreadCount,
 		ProgressSink& progressSink
 	) const override;

rhubarb/src/recognition/PocketSphinxRecognizer.cpp

@@ -6,8 +6,10 @@
 #include "languageModels.h"
 #include "tokenization.h"
 #include "g2p.h"
+#include "audio/DcOffset.h"
 #include "time/ContinuousTimeline.h"
 #include "audio/processing.h"
+#include "audio/voiceActivityDetection.h"
 #include "time/timedLogging.h"
 
 extern "C" {
@@ -334,9 +336,41 @@ static Timeline<Phone> utteranceToPhones(
 BoundedTimeline<Phone> PocketSphinxRecognizer::recognizePhones(
 	const AudioClip& inputAudioClip,
 	optional<std::string> dialog,
+	optional<BoundedTimeline<Phone>> alignedPhones,
 	int maxThreadCount,
 	ProgressSink& progressSink
 ) const {
+	if (alignedPhones) {
+		// Make sure audio stream has no DC offset
+		const unique_ptr<AudioClip> audioClip = inputAudioClip.clone() | removeDcOffset();
+
+		// Split audio into utterances
+		JoiningBoundedTimeline<void> utterances;
+		try {
+			utterances = detectVoiceActivity(*audioClip, progressSink);
+		} catch (...) {
+			std::throw_with_nested(runtime_error("Error detecting segments of speech."));
+		}
+
+		BoundedTimeline<Phone> result(utterances.getRange());
+		for (auto& utterance : utterances) {
+			// Copy over utterance phones
+			BoundedTimeline<Phone> utteranceResult(utterance.getTimeRange(), *alignedPhones);
+
+			// Guess positions of noise sounds
+			const JoiningTimeline<void> noiseSounds = getNoiseSounds(utteranceResult.getRange(), utteranceResult);
+			for (const auto& noiseSound : noiseSounds) {
+				utteranceResult.set(noiseSound.getTimeRange(), Phone::Noise);
+			}
+
+			for (const auto& timedValue : utteranceResult) {
+				result.set(timedValue);
+			}
+		}
+
+		return result;
+	}
+
 	return ::recognizePhones(
 		inputAudioClip, dialog, &createDecoder, &utteranceToPhones, maxThreadCount, progressSink);
 }

rhubarb/src/recognition/PocketSphinxRecognizer.h

@@ -8,6 +8,7 @@ public:
 	BoundedTimeline<Phone> recognizePhones(
 		const AudioClip& inputAudioClip,
 		boost::optional<std::string> dialog,
+		boost::optional<BoundedTimeline<Phone>> alignedPhones,
 		int maxThreadCount,
 		ProgressSink& progressSink
 	) const override;

rhubarb/src/recognition/Recognizer.h

@@ -12,6 +12,7 @@ public:
 	virtual BoundedTimeline<Phone> recognizePhones(
 		const AudioClip& audioClip,
 		boost::optional<std::string> dialog,
+		boost::optional<BoundedTimeline<Phone>> alignedPhones,
 		int maxThreadCount,
 		ProgressSink& progressSink
 	) const = 0;

rhubarb/src/rhubarb/main.cpp

@@ -115,6 +115,33 @@ ShapeSet getTargetShapeSet(const string& extendedShapesString) {
 	return result;
 }
 
+BoundedTimeline<Phone> readAlignmentFile(const path& filePath) {
+	if (!exists(filePath)) {
+		throw std::invalid_argument(fmt::format("File {} does not exist.", filePath.u8string()));
+	}
+	try {
+		std::ifstream file;
+		file.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+		file.open(filePath);
+		file.exceptions(0);
+		Timeline<Phone> result;
+		while (file) {
+			double start, end;
+			Phone phone;
+			file >> start >> end >> phone;
+			result.set(
+				centiseconds(static_cast<int>(start * 100)),
+				centiseconds(static_cast<int>(end * 100)),
+				phone
+			);
+		}
+
+		return BoundedTimeline<Phone>(result.getRange(), result);
+	} catch (...) {
+		std::throw_with_nested(std::runtime_error(fmt::format("Error reading file {0}.", filePath.u8string())));
+	}
+}
+
 int main(int platformArgc, char* platformArgv[]) {
 	// Set up default logging so early errors are printed to stdout
 	const logging::Level defaultMinStderrLevel = logging::Level::Error;
@@ -174,6 +201,11 @@ int main(int platformArgc, char* platformArgv[]) {
 		false, "GHX", "string", cmd
 	);
 
+	tclap::ValueArg<string> alignmentFile(
+		"", "alignmentFile", "A .tsv file containing pre-calculated phoneme alignment data.",
+		false, string(), "string", cmd
+	);
+
 	tclap::ValueArg<string> dialogFile(
 		"d", "dialogFile", "A file containing the text of the dialog.",
 		false, string(), "string", cmd
@@ -263,6 +295,9 @@ int main(int platformArgc, char* platformArgv[]) {
 				dialogFile.isSet()
 					? readUtf8File(u8path(dialogFile.getValue()))
 					: boost::optional<string>(),
+				alignmentFile.isSet()
+					? readAlignmentFile(u8path(alignmentFile.getValue()))
+					: boost::optional<BoundedTimeline<Phone>>(),
 				*createRecognizer(recognizerType.getValue()),
 				targetShapeSet,
 				maxThreadCount.getValue(),