From 3bc4384b44bed5fce1a69eee970b41e55eaebb36 Mon Sep 17 00:00:00 2001 From: Daniel Wolf Date: Thu, 29 Dec 2016 20:45:53 +0100 Subject: [PATCH] Added overarching animation step that prevents long static segments See http://animateducated.blogspot.com/2016/10/lip-sync-animation-2.html?showComment=1478861729702#c2940729096183546458 --- CMakeLists.txt | 3 + src/animation/mouthAnimation.cpp | 19 +-- src/animation/staticSegments.cpp | 197 +++++++++++++++++++++++++++++++ src/animation/staticSegments.h | 14 +++ src/tools/nextCombination.h | 47 ++++++++ 5 files changed, 273 insertions(+), 7 deletions(-) create mode 100644 src/animation/staticSegments.cpp create mode 100644 src/animation/staticSegments.h create mode 100644 src/tools/nextCombination.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 64bd0c8..8bd57dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,6 +229,8 @@ add_library(rhubarb-animation src/animation/ShapeRule.cpp src/animation/ShapeRule.h src/animation/shapeShorthands.h + src/animation/staticSegments.cpp + src/animation/staticSegments.h src/animation/targetShapeSet.cpp src/animation/targetShapeSet.h src/animation/timingOptimization.cpp @@ -375,6 +377,7 @@ add_library(rhubarb-tools src/tools/exceptions.cpp src/tools/exceptions.h src/tools/Lazy.h + src/tools/nextCombination.h src/tools/NiceCmdLineOutput.cpp src/tools/NiceCmdLineOutput.h src/tools/ObjectPool.h diff --git a/src/animation/mouthAnimation.cpp b/src/animation/mouthAnimation.cpp index e6c62ae..5ff7855 100644 --- a/src/animation/mouthAnimation.cpp +++ b/src/animation/mouthAnimation.cpp @@ -6,6 +6,7 @@ #include "tweening.h" #include "timingOptimization.h" #include "targetShapeSet.h" +#include "staticSegments.h" JoiningContinuousTimeline animate(const BoundedTimeline &phones, const ShapeSet& targetShapeSet) { // Create timeline of shape rules @@ -17,15 +18,19 @@ JoiningContinuousTimeline animate(const BoundedTimeline &phones, c shapeRules = convertToTargetShapeSet(shapeRules, targetShapeSetPlusX); // Animate in multiple steps - JoiningContinuousTimeline animation = animateRough(shapeRules); - animation = optimizeTiming(animation); - animation = animatePauses(animation); - animation = insertTweens(animation); - animation = convertToTargetShapeSet(animation, targetShapeSet); + auto performMainAnimationSteps = [&targetShapeSet](const auto& shapeRules) { + JoiningContinuousTimeline animation = animateRough(shapeRules); + animation = optimizeTiming(animation); + animation = animatePauses(animation); + animation = insertTweens(animation); + animation = convertToTargetShapeSet(animation, targetShapeSet); + return animation; + }; + const JoiningContinuousTimeline result = avoidStaticSegments(shapeRules, performMainAnimationSteps); - for (const auto& timedShape : animation) { + for (const auto& timedShape : result) { logTimedEvent("shape", timedShape); } - return animation; + return result; } diff --git a/src/animation/staticSegments.cpp b/src/animation/staticSegments.cpp new file mode 100644 index 0000000..2225612 --- /dev/null +++ b/src/animation/staticSegments.cpp @@ -0,0 +1,197 @@ +#include "staticSegments.h" +#include +#include +#include "nextCombination.h" + +using std::vector; +using boost::optional; + +int getSyllableCount(const ContinuousTimeline& shapeRules, TimeRange timeRange) { + if (timeRange.empty()) return 0; + + const auto begin = shapeRules.find(timeRange.getStart()); + const auto end = std::next(shapeRules.find(timeRange.getEnd(), FindMode::SampleLeft)); + + // Treat every vowel as one syllable + int syllableCount = 0; + for (auto it = begin; it != end; ++it) { + const ShapeRule shapeRule = it->getValue(); + + // Disregard phones that are mostly outside the specified time range. + const centiseconds phoneMiddle = shapeRule.phoneTiming.getMiddle(); + if (phoneMiddle < timeRange.getStart() || phoneMiddle >= timeRange.getEnd()) continue; + + auto phone = shapeRule.phone; + if (phone && isVowel(*phone)) { + ++syllableCount; + } + } + + return syllableCount; +} + +// A static segment is a prolonged period during which the mouth shape doesn't change +vector getStaticSegments(const ContinuousTimeline& shapeRules, const JoiningContinuousTimeline& animation) { + // A static segment must contain a certain number of syllables to look distractingly static + const int minSyllableCount = 3; + // It must also have a minimum duration. The same number of syllables in fast speech usually looks good. + const centiseconds minDuration = 75_cs; + + vector result; + for (const auto& timedShape : animation) { + const TimeRange timeRange = timedShape.getTimeRange(); + if (timeRange.getDuration() >= minDuration && getSyllableCount(shapeRules, timeRange) >= minSyllableCount) { + result.push_back(timeRange); + } + } + + return result; +} + +// Indicates whether this shape rule can potentially be replaced by a modified version that breaks up long static segments +bool canChange(const ShapeRule& rule) { + return rule.phone && isVowel(*rule.phone) && rule.shapeSet.size() == 1; +} + +// Returns a new shape rule that is identical to the specified one, except that it leads to a slightly different visualization +ShapeRule getChangedShapeRule(const ShapeRule& rule) { + assert(canChange(rule)); + + ShapeRule result(rule); + // So far, I've only encountered B as a static shape. + // If there is ever a problem with another static shape, this function can easily be extended. + if (rule.shapeSet == ShapeSet{Shape::B}) { + result.shapeSet = {Shape::C}; + } + return result; +} + +// Contains the start times of all rules to be changed +using RuleChanges = vector; + +// Replaces the indicated shape rules with slightly different ones, breaking up long static segments +ContinuousTimeline applyChanges(const ContinuousTimeline& shapeRules, const RuleChanges& changes) { + ContinuousTimeline result(shapeRules); + for (centiseconds changedRuleStart : changes) { + const Timed timedOriginalRule = *shapeRules.get(changedRuleStart); + const ShapeRule changedRule = getChangedShapeRule(timedOriginalRule.getValue()); + result.set(timedOriginalRule.getTimeRange(), changedRule); + } + return result; +} + +class RuleChangeScenario { +public: + RuleChangeScenario( + const ContinuousTimeline& originalRules, + const RuleChanges& changes, + AnimationFunction animate) : + changedRules(applyChanges(originalRules, changes)), + animation(animate(changedRules)), + staticSegments(getStaticSegments(changedRules, animation)) {} + + bool isBetterThan(const RuleChangeScenario& rhs) const { + // We want zero static segments + if (staticSegments.size() == 0 && rhs.staticSegments.size() > 0) return true; + + // Short shapes are better than long ones. Minimize sum-of-squares. + if (getSumOfShapeDurationSquares() < rhs.getSumOfShapeDurationSquares()) return true; + + return false; + } + + int getStaticSegmentCount() const { + return staticSegments.size(); + } + + ContinuousTimeline getChangedRules() const { + return changedRules; + } + +private: + ContinuousTimeline changedRules; + JoiningContinuousTimeline animation; + vector staticSegments; + + double getSumOfShapeDurationSquares() const { + return std::accumulate(animation.begin(), animation.end(), 0.0, [](const double sum, const Timed& timedShape) { + const double duration = std::chrono::duration_cast>(timedShape.getDuration()).count(); + return sum + duration * duration; + }); + } +}; + +RuleChanges getPossibleRuleChanges(const ContinuousTimeline& shapeRules) { + RuleChanges result; + for (auto it = shapeRules.begin(); it != shapeRules.end(); ++it) { + const ShapeRule rule = it->getValue(); + if (canChange(rule)) { + result.push_back(it->getStart()); + } + } + return result; +} + +ContinuousTimeline fixStaticSegmentRules(const ContinuousTimeline& shapeRules, AnimationFunction animate) { + // The complexity of this function is exponential with the number of replacements. So let's cap that value. + const int maxReplacementCount = 3; + + // All potential changes + const RuleChanges possibleRuleChanges = getPossibleRuleChanges(shapeRules); + + // Find best solution. Start with a single replacement, then increase as necessary. + RuleChangeScenario bestScenario(shapeRules, {}, animate); + for (int replacementCount = 1; bestScenario.getStaticSegmentCount() > 0 && replacementCount <= maxReplacementCount; ++replacementCount) { + // Only the first elements of `currentRuleChanges` count + auto currentRuleChanges(possibleRuleChanges); + do { + RuleChangeScenario currentScenario(shapeRules, {currentRuleChanges.begin(), currentRuleChanges.begin() + replacementCount}, animate); + if (currentScenario.isBetterThan(bestScenario)) { + bestScenario = currentScenario; + } + } while (next_combination(currentRuleChanges.begin(), currentRuleChanges.begin() + replacementCount, currentRuleChanges.end())); + } + + return bestScenario.getChangedRules(); +} + +// Indicates whether the specified shape rule may result in different shapes depending on context +bool isFlexible(const ShapeRule& rule) { + return rule.shapeSet.size() > 1; +} + +// Extends the specified time range until it starts and ends with a non-flexible shape rule, if possible +TimeRange extendToFixedRules(const TimeRange& timeRange, const ContinuousTimeline& shapeRules) { + auto first = shapeRules.find(timeRange.getStart()); + while (first != shapeRules.begin() && isFlexible(first->getValue())) { + --first; + } + auto last = shapeRules.find(timeRange.getEnd(), FindMode::SampleLeft); + while (std::next(last) != shapeRules.end() && isFlexible(last->getValue())) { + ++last; + } + return TimeRange(first->getStart(), last->getEnd()); +} + +JoiningContinuousTimeline avoidStaticSegments(const ContinuousTimeline& shapeRules, AnimationFunction animate) { + const auto animation = animate(shapeRules); + const vector staticSegments = getStaticSegments(shapeRules, animation); + if (staticSegments.empty()) { + return animation; + } + + // Modify shape rules to eliminate static segments + ContinuousTimeline fixedShapeRules(shapeRules); + for (const TimeRange& staticSegment : staticSegments) { + // Extend time range to the left and right so we don't lose adjacent rules that might influence the animation + const TimeRange extendedStaticSegment = extendToFixedRules(staticSegment, shapeRules); + + // Fix shape rules within the static segment + const auto fixedSegmentShapeRules = fixStaticSegmentRules({extendedStaticSegment, ShapeRule::getInvalid(), fixedShapeRules}, animate); + for (const auto& timedShapeRule : fixedSegmentShapeRules) { + fixedShapeRules.set(timedShapeRule); + } + } + + return animate(fixedShapeRules); +} diff --git a/src/animation/staticSegments.h b/src/animation/staticSegments.h new file mode 100644 index 0000000..cd7f471 --- /dev/null +++ b/src/animation/staticSegments.h @@ -0,0 +1,14 @@ +#pragma once + +#include "Shape.h" +#include "ContinuousTimeline.h" +#include "ShapeRule.h" +#include + +using AnimationFunction = std::function(const ContinuousTimeline&)>; + +// Calls the specified animation function with the specified shape rules. +// If the resulting animation contains long static segments, the shape rules are tweaked and animated again. +// Static segments happen rather often. +// See http://animateducated.blogspot.de/2016/10/lip-sync-animation-2.html?showComment=1478861729702#c2940729096183546458. +JoiningContinuousTimeline avoidStaticSegments(const ContinuousTimeline& shapeRules, AnimationFunction animate); diff --git a/src/tools/nextCombination.h b/src/tools/nextCombination.h new file mode 100644 index 0000000..ca39ae7 --- /dev/null +++ b/src/tools/nextCombination.h @@ -0,0 +1,47 @@ +#pragma once + +#include + +// next_combination Template +// Originally written by Thomas Draper +// +// Designed after next_permutation in STL +// Inspired by Mark Nelson's article http://www.dogma.net/markn/articles/Permutations/ +// +// Start with a sorted container with thee iterators -- first, k, last +// After each iteration, the first k elements of the container will be +// a combination. When there are no more combinations, the container +// will return to the original sorted order. +template +inline bool next_combination(const Iterator first, Iterator k, const Iterator last) { + // Handle degenerate cases + if (first == last || std::next(first) == last || first == k || k == last) { + return false; + } + + Iterator it1 = k; + Iterator it2 = std::prev(last); + // Search down to find first comb entry less than final entry + while (it1 != first) { + --it1; + if (*it1 < *it2) { + Iterator j = k; + while (!(*it1 < *j)) { + ++j; + } + std::iter_swap(it1, j); + ++it1; + ++j; + it2 = k; + std::rotate(it1, j, last); + while (last != j) { + ++j; + ++it2; + } + std::rotate(k, it2, last); + return true; + } + } + std::rotate(first, k, last); + return false; +}