From ea4bab70362d2d41977b8ea96ebc8b3f8229d53f Mon Sep 17 00:00:00 2001
From: pnossol <patrick.nossol@gmail.com>
Date: Tue, 31 May 2022 09:27:49 +0200
Subject: [PATCH] Implemented preprocessing of data and other tweaks
---
Content/MoCapMap.umap | 4 +-
Content/SaveSequenceRig.uasset | 4 +-
Source/MoCapPlugin/Private/MCController.cpp | 541 ++++++++++++++++++--
Source/MoCapPlugin/Private/MCDefines.cpp | 38 ++
Source/MoCapPlugin/Private/MCPawn.cpp | 54 +-
Source/MoCapPlugin/Private/MCRigUnits.cpp | 67 ++-
Source/MoCapPlugin/Public/MCController.h | 18 +-
Source/MoCapPlugin/Public/MCDefines.h | 4 +
Source/MoCapPlugin/Public/MCPawn.h | 4 +
Source/MoCapPlugin/Public/MCRigUnits.h | 27 +-
10 files changed, 706 insertions(+), 55 deletions(-)
create mode 100644 Source/MoCapPlugin/Private/MCDefines.cpp
diff --git a/Content/MoCapMap.umap b/Content/MoCapMap.umap
index 4fd773a..6d1167a 100644
--- a/Content/MoCapMap.umap
+++ b/Content/MoCapMap.umap
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:247c65dcafd56213adf01bf90c1e7fa6afd34c321deec48c8f618e658f5d0237
-size 39920
+oid sha256:92a5fa84a7731e65598ce589a146da254ef412238115d8187d6ec7ab5c455170
+size 39985
diff --git a/Content/SaveSequenceRig.uasset b/Content/SaveSequenceRig.uasset
index 6bc7ca4..dc8dfa1 100644
--- a/Content/SaveSequenceRig.uasset
+++ b/Content/SaveSequenceRig.uasset
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:29a5de877c6fd233d0416e428dd532fed364376333c5fd7d47eba29a07779288
-size 4080636
+oid sha256:62d3c42edf0f9d2db2d91756dcc93681e4c7f7d801973e757ae9840ae0efbc4a
+size 4560157
diff --git a/Source/MoCapPlugin/Private/MCController.cpp b/Source/MoCapPlugin/Private/MCController.cpp
index 25314bd..d2a6025 100644
--- a/Source/MoCapPlugin/Private/MCController.cpp
+++ b/Source/MoCapPlugin/Private/MCController.cpp
@@ -136,7 +136,7 @@ void AMCController::SaveToAnimMode() {
}
if (!AnimSaveState.WaitForAnimInstance) {
- if (AnimSaveState.CurrentEntryIndex < AnimSaveState.StringData.Num()) {
+ if (AnimSaveState.CurrentEntryIndex < AnimSaveState.AnimData.Num()) {
InputNextFrame();
}
else {
@@ -155,83 +155,552 @@ void AMCController::SaveToAnimMode() {
}
-void AMCController::InputNextFrame() {
+void AMCController::ScaleAnimDataInterval(int start, int end, float DeltaTime) {
+
+ FTimespan StartTime = AnimSaveState.AnimData[start].Timestamp;
+ float OldTime = (AnimSaveState.AnimData[end].Timestamp - AnimSaveState.AnimData[start].Timestamp).GetTotalSeconds();
+ float NewTime = OldTime + DeltaTime;
+ float ScaleFactor = NewTime / OldTime;
+ int OldNrFrames = end + 1 - start;
+ int NewNrFrames = ScaleFactor * OldNrFrames;
- bool stop = false;
- UMCAnimInstance* AI = AnimSaveState.Pawn->GetAnimInstance();
+ TArray<FProcessedAnimData> OldAnimData;
+ for (int i = 0; i < AnimSaveState.AnimData.Num(); i++) {
+ OldAnimData.Push(AnimSaveState.AnimData[i]);
+ }
- for (; !stop && AnimSaveState.CurrentEntryIndex < AnimSaveState.StringData.Num();) {
+ for (int i = end + 1; i < AnimSaveState.AnimData.Num(); i++) {
+ AnimSaveState.AnimData[i].Timestamp += FTimespan::FromSeconds(DeltaTime);
+ }
- auto Entry = AnimSaveState.StringData[AnimSaveState.CurrentEntryIndex];
+ AnimSaveState.AnimData.RemoveAt(start, OldNrFrames, true);
- FString TimeString, JsonString;
- Entry.Split(" ", &TimeString, &JsonString);
- FTimespan Timestamp = MCUtils::StringToTimespan(TimeString);
+ int CurOldIndex = start;
- // Check if Timestemp went over the 1 hour mark
- if (Timestamp < AnimSaveState.LastMarker) {
- Timestamp = Timestamp + FTimespan::FromHours(1);
+ for (int i = 0; i < NewNrFrames; i++) {
+
+ FProcessedAnimData Data;
+ Data.Timestamp = StartTime + FTimespan::FromSeconds((float)i * NewTime / (float)NewNrFrames);
+ //SmoothStop
+ //float TransformedI = 1.f - FMath::Pow(1.f - ((float)i / (float)NewNrFrames), 4);
+ //SmoothStart
+ float TransformedI = FMath::Pow(((float)i / (float)NewNrFrames), 4);
+ FTimespan OldTimestamp = StartTime + FTimespan::FromSeconds(TransformedI * OldTime);
+
+ bool stop = false;
+ while (CurOldIndex + 1 < OldAnimData.Num() && !(OldTimestamp >= OldAnimData[CurOldIndex].Timestamp && OldTimestamp <= OldAnimData[CurOldIndex + 1].Timestamp)) {
+ CurOldIndex++;
}
- // Check if next frame should be taken
- if (AnimSaveState.CurrentMarker > 0 && Timestamp > AnimSaveState.NextFrame) {
+ float AlphaTime;
+ if (CurOldIndex + 1 >= OldAnimData.Num() || OldAnimData[CurOldIndex + 1].IsEnd || OldAnimData[CurOldIndex + 1].IsMarker) {
+ AlphaTime = 0.f;
+ }
+ else if (OldAnimData[CurOldIndex].IsEnd || OldAnimData[CurOldIndex].IsMarker) {
+ AlphaTime = 1.f;
+ }
+ else {
+ AlphaTime = (OldTimestamp - OldAnimData[CurOldIndex].Timestamp).GetTotalSeconds() / (OldAnimData[CurOldIndex + 1].Timestamp - OldAnimData[CurOldIndex].Timestamp).GetTotalSeconds();
+ }
- AnimSaveState.Pawn->AICalcFrame();
- AnimSaveState.NextFrame = AnimSaveState.NextFrame + FTimespan::FromSeconds(AnimSaveState.SPF);
- stop = true;
- AnimSaveState.WaitForAnimInstance = true;
+ //process sensor data
+ FSensorData& SensorData = Data.SensorData;
+ for (int j = 0; j < EBodyPart::LAST; j++) {
+
+ EBodyPart Type = EBodyPart(j);
+ FSensorDataEntry* Entry = SensorData.GetEntry(Type);
+
+ const FSensorDataEntry* OldEntry0 = OldAnimData[CurOldIndex].SensorData.GetEntry(Type);
+ if (AlphaTime > 0.f) {
+
+ const FSensorDataEntry* OldEntry1 = OldAnimData[CurOldIndex + 1].SensorData.GetEntry(Type);
+
+ Entry->Pos = (1.f - AlphaTime) * OldEntry0->Pos + AlphaTime * OldEntry1->Pos;
+ Entry->Rot = FQuat::Slerp(OldEntry0->Rot, OldEntry1->Rot, AlphaTime);
+ Entry->valid = true;
+ Entry->PosUsed = true;
+
+ }
+ else {
+ Entry->Pos = OldEntry0->Pos;
+ Entry->Rot = OldEntry0->Rot;
+ Entry->valid = true;
+ Entry->PosUsed = true;
+ }
+ }
+
+ //process finger data
+ if (bFingerTrackingEnabled) {
+
+ FFingerData& FingerData = Data.FingerData;
+ for (int f = 0; f < 2; f++) {
+
+ FFingerDataEntry* Hand = f == 0 ? &FingerData.LeftHand : &FingerData.RightHand;
+ FFingerDataEntry* OldHand0 = f == 0 ? &OldAnimData[CurOldIndex].FingerData.LeftHand : &OldAnimData[CurOldIndex].FingerData.RightHand;
+ FFingerDataEntry* OldHand1 = AlphaTime > 0.f ? (f == 0 ? &OldAnimData[CurOldIndex + 1].FingerData.LeftHand : &OldAnimData[CurOldIndex + 1].FingerData.RightHand) : (&OldAnimData[CurOldIndex].FingerData.RightHand);
+
+ Hand->Thumb = (1.f - AlphaTime) * OldHand0->Thumb + AlphaTime * OldHand1->Thumb;
+ Hand->Index = (1.f - AlphaTime) * OldHand0->Index + AlphaTime * OldHand1->Index;
+ Hand->Middle = (1.f - AlphaTime) * OldHand0->Middle + AlphaTime * OldHand1->Middle;
+ Hand->Ring = (1.f - AlphaTime) * OldHand0->Ring + AlphaTime * OldHand1->Ring;
+ Hand->Pinky = (1.f - AlphaTime) * OldHand0->Pinky + AlphaTime * OldHand1->Pinky;
+ Hand->Thumb_Index = (1.f - AlphaTime) * OldHand0->Thumb_Index + AlphaTime * OldHand1->Thumb_Index;
+ Hand->Index_Middle = (1.f - AlphaTime) * OldHand0->Index_Middle + AlphaTime * OldHand1->Index_Middle;
+ Hand->Middle_Ring = (1.f - AlphaTime) * OldHand0->Middle_Ring + AlphaTime * OldHand1->Middle_Ring;
+ Hand->Ring_Pinky = (1.f - AlphaTime) * OldHand0->Ring_Pinky + AlphaTime * OldHand1->Ring_Pinky;
+
+ }
- FeedbackWidget->ProgBar->SetPercent((float)AnimSaveState.CurrentEntryIndex / (float)AnimSaveState.StringData.Num());
+ }
+
+ AnimSaveState.AnimData.Insert(Data, start + i);
+
+ }
+
+}
+void AMCController::PreprocessRecording() {
+
+ UMCAnimInstance* AI = AnimSaveState.Pawn->GetAnimInstance();
+
+ //----translate json data to AnimSaveState.AnimData----
+
+ FTimespan LastStamp;
+ FTimespan NextFrame;
+ int CurrentMarker = 0;
+ float MaxPelvisSpineLength = -1.f;
+
+ for (int i = 0; i < AnimSaveState.StringData.Num(); i++) {
+
+ auto Entry = AnimSaveState.StringData[i];
+
+ FString TimeString, JsonString;
+ Entry.Split(" ", &TimeString, &JsonString);
+ FTimespan Timestamp = MCUtils::StringToTimespan(TimeString);
+
+ if (Timestamp < LastStamp) {
+ Timestamp = Timestamp + FTimespan::FromHours(1);
}
TSharedPtr<FJsonObject> TmpJson = MCUtils::StringToJson(JsonString);
if (!TmpJson->HasField("Type")) {
- AnimSaveState.CurrentEntryIndex++;
continue;
}
FString Type = TmpJson->GetStringField("Type");
if (Type == "Start") {
- AnimSaveState.LastMarker = Timestamp;
- AnimSaveState.NextFrame = Timestamp;
+ LastStamp = Timestamp;
+ NextFrame = Timestamp;
}
else if (Type == "Marker" || Type == "End") {
- AnimSaveState.LastMarker = Timestamp;
- AnimSaveState.NextFrame = Timestamp;
+ LastStamp = Timestamp;
+ NextFrame = Timestamp;
+
+ AnimSaveState.AnimData.Push(FProcessedAnimData());
+ AnimSaveState.AnimData.Last().Timestamp = Timestamp;
+ if (Type == "Marker") {
+ AnimSaveState.AnimData.Last().IsMarker = true;
+ CurrentMarker++;
+ }
+ else {
+ AnimSaveState.AnimData.Last().IsEnd = true;
+ }
+
+ }
+ else if (Type == "ViveData") {
+
+ AnimSaveState.Pawn->InputViveDataToAnimInstance(TmpJson);
+ if (bFingerTrackingEnabled) {
+ AnimSaveState.Pawn->InputFingerDataToAnimInstance(TmpJson);
+ }
+
+ if (CurrentMarker > 0 && Timestamp > NextFrame) {
+
+ AnimSaveState.AnimData.Push(FProcessedAnimData());
+ AnimSaveState.AnimData.Last().Timestamp = Timestamp;
+ AnimSaveState.AnimData.Last().SensorData = AI->SensorData;
+ if (bFingerTrackingEnabled) {
+ AnimSaveState.AnimData.Last().FingerData = AI->FingerData;
+ }
+
+ NextFrame = NextFrame + FTimespan::FromSeconds(AnimSaveState.SPF);
+
+ }
+
+ }
+ else if (Type == "Offsets") {
+ AnimSaveState.Pawn->InputViveOffsetsToAnimInstance(TmpJson);
+ }
+
+ }
+
+ //----pull down hip amount----
+
+ float PullDownHip = 0;
+ {
+ FVector OffsetPos = AnimSaveState.AnimData[0].SensorData.LowerBody.Pos;
+ float diff = OffsetPos.Z;
+ FQuat OffsetRot = AnimSaveState.AnimData[0].SensorData.LowerBody.Rot;
+ OffsetRot = OffsetRot * FQuat(FVector(0, 0, 1), FMath::DegreesToRadians(180));
+ AnimSaveState.Pawn->OffsetSensorData(EBodyPart::LowerBody, OffsetPos, OffsetRot);
+ diff = OffsetPos.Z - diff;
+ PullDownHip = diff;
+ }
+
+ //----process the anim data----
+
+ //---Sensor turned off -> interpolate
+ for (int i = 0; i < AnimSaveState.AnimData.Num(); i++) {
+
+ FProcessedAnimData& AnimData = AnimSaveState.AnimData[i];
+
+ if (AnimData.IsMarker || AnimData.IsEnd) {
+ continue;
+ }
+
+ //process sensor data
+ FSensorData& SensorData = AnimData.SensorData;
+ for (int j = 0; j < EBodyPart::LAST; j++) {
+
+ EBodyPart Type = EBodyPart(j);
+ FSensorDataEntry* Entry = SensorData.GetEntry(Type);
+
+ if (Entry->Pos.IsNearlyZero() && Entry->Rot.Rotator().IsNearlyZero()) {
+
+ FVector StartPos = Entry->Pos;
+ FQuat StartRot = Entry->Rot;
+ int StartIndex = i;
+ if (i > 0) {
+ StartPos = AnimSaveState.AnimData[i - 1].SensorData.GetEntry(Type)->Pos;
+ StartRot = AnimSaveState.AnimData[i - 1].SensorData.GetEntry(Type)->Rot;
+ StartIndex = i - 1;
+ }
+
+ FVector EndPos;
+ FQuat EndRot;
+ int EndIndex = AnimSaveState.AnimData.Num() - 1;
+ for (int k = i + 1; k < AnimSaveState.AnimData.Num(); k++) {
+ FSensorDataEntry* CurEntry = AnimSaveState.AnimData[k].SensorData.GetEntry(Type);
+ if (!CurEntry->Pos.IsNearlyZero() || !CurEntry->Rot.Rotator().IsNearlyZero()) {
+ EndPos = CurEntry->Pos;
+ EndRot = CurEntry->Rot;
+ EndIndex = k;
+ break;
+ }
+ }
+
+ float alphaStep = 1.f / (float)(EndIndex - StartIndex);
+ float alpha = alphaStep;
+ for (int k = StartIndex + 1; k < EndIndex; k++) {
+
+ FSensorDataEntry* CurEntry = AnimSaveState.AnimData[k].SensorData.GetEntry(Type);
+ CurEntry->Pos = (1.f - alpha) * StartPos + alpha * EndPos;
+ CurEntry->Rot = FQuat::Slerp(StartRot, EndRot, alpha);
+
+ //process finger data
+ if ((Type == EBodyPart::HandL || Type == EBodyPart::HandR) && bFingerTrackingEnabled) {
+
+ FFingerData& FingerData = AnimData.FingerData;
+ for (int f = 0; f < 2; f++) {
+
+ FFingerDataEntry* Hand = f == 0 ? &FingerData.LeftHand : &FingerData.RightHand;
+ FFingerDataEntry* HandStart = f == 0 ? &AnimSaveState.AnimData[StartIndex].FingerData.LeftHand : &AnimSaveState.AnimData[StartIndex].FingerData.RightHand;
+ FFingerDataEntry* HandEnd = f == 0 ? &AnimSaveState.AnimData[EndIndex].FingerData.LeftHand : &AnimSaveState.AnimData[EndIndex].FingerData.RightHand;
+
+ Hand->Thumb = (1.f - alpha) * HandStart->Thumb + alpha * HandEnd->Thumb;
+ Hand->Index = (1.f - alpha) * HandStart->Index + alpha * HandEnd->Index;
+ Hand->Middle = (1.f - alpha) * HandStart->Middle + alpha * HandEnd->Middle;
+ Hand->Ring = (1.f - alpha) * HandStart->Ring + alpha * HandEnd->Ring;
+ Hand->Pinky = (1.f - alpha) * HandStart->Pinky + alpha * HandEnd->Pinky;
+ Hand->Thumb_Index = (1.f - alpha) * HandStart->Thumb_Index + alpha * HandEnd->Thumb_Index;
+ Hand->Index_Middle = (1.f - alpha) * HandStart->Index_Middle + alpha * HandEnd->Index_Middle;
+ Hand->Middle_Ring = (1.f - alpha) * HandStart->Middle_Ring + alpha * HandEnd->Middle_Ring;
+ Hand->Ring_Pinky = (1.f - alpha) * HandStart->Ring_Pinky + alpha * HandEnd->Ring_Pinky;
+
+ }
+
+ }
+
+ alpha += alphaStep;
+
+ }
+
+
+ }
+
+ }
+
+ }
+
+ //---other processing---
+ for (int i = 0; i < AnimSaveState.AnimData.Num(); i++) {
+
+ FProcessedAnimData& AnimData = AnimSaveState.AnimData[i];
+
+ if (AnimData.IsMarker || AnimData.IsEnd) {
+ continue;
+ }
+
+ //process sensor data
+ FSensorData& SensorData = AnimData.SensorData;
+ for (int j = 0; j < EBodyPart::LAST; j++) {
+
+ EBodyPart Type = EBodyPart(j);
+ FSensorDataEntry* Entry = SensorData.GetEntry(Type);
+
+ //pull down hip
+ if (Type == EBodyPart::LowerBody) {
+ FQuat Swing, Twist;
+ Entry->Rot.ToSwingTwist(FVector(0, 0, 1), Swing, Twist);
+ float angle = FMath::RadiansToDegrees(Swing.GetAngle());
+ if (angle > 180) {
+ angle -= 360;
+ }
+ //Entry.Pos.Z += PullDownHip * FMath::Abs(angle / 180.f);
+ }
+
+ //analyse z pos of feet
+ if (Type == EBodyPart::LowerLegL) {
+
+ int window = 5;
+ bool isMin = true;
+ for (int w = 1; w <= window; w++) {
+ if (i + w < AnimSaveState.AnimData.Num()) {
+ FVector Pos = AnimSaveState.AnimData[i + w].SensorData.LowerLegL.Pos;
+ if (Entry->Pos.Z <= Pos.Z) {
+ isMin = false;
+ break;
+ }
+ }
+ if (i - w >= 0) {
+ FVector Pos = AnimSaveState.AnimData[i - w].SensorData.LowerLegL.Pos;
+ if (Entry->Pos.Z <= Pos.Z) {
+ isMin = false;
+ break;
+ }
+ }
+ }
+
+ if (isMin) {
+ AnimSaveState.LowestPoints.Push(Entry->Pos.Z);
+ }
+
+ //offset
+
+ FVector OffsetPos = Entry->Pos;
+ FQuat OffsetRot = Entry->Rot;
+ AnimSaveState.Pawn->OffsetSensorData(Type, OffsetPos, OffsetRot);
+
+ isMin = true;
+ for (int w = 1; w <= window; w++) {
+ if (i + w < AnimSaveState.AnimData.Num()) {
+ FVector Pos = AnimSaveState.AnimData[i + w].SensorData.LowerLegL.Pos;
+ if (OffsetPos.Z <= Pos.Z) {
+ isMin = false;
+ break;
+ }
+ }
+ if (i - w >= 0) {
+ FVector Pos = AnimSaveState.AnimData[i - w].SensorData.LowerLegL.Pos;
+ if (OffsetPos.Z <= Pos.Z) {
+ isMin = false;
+ break;
+ }
+ }
+ }
+
+ if (isMin) {
+ AnimSaveState.LowestPointsOffset.Push(OffsetPos.Z);
+ }
+
+ }
+
+ }
+
+ //process finger data
+ if (bFingerTrackingEnabled) {
+
+ }
+
+ }
+
+ //---find last slow part of the gesture and scale it up---
+
+ TArray<float> GestureSpeed;
+
+ if (GestureHoldExcessTime > 0.f) {
+
+
+
+ for (int i = 1; i < AnimSaveState.AnimData.Num(); i++) {
+
+ FProcessedAnimData& AnimData = AnimSaveState.AnimData[i];
+
+ if (AnimData.IsMarker || AnimData.IsEnd) {
+ continue;
+ }
+
+ //process sensor data
+
+ float Speed = 0.f;
+ int SpeedCount = 0;
+ int window = 1;
+
+ FSensorData& SensorData = AnimData.SensorData;
+ for (int j = 0; j < EBodyPart::LAST; j++) {
+
+ EBodyPart Type = EBodyPart(j);
+ FSensorDataEntry* Entry = SensorData.GetEntry(Type);
+
+ if (Type == EBodyPart::HandL || Type == EBodyPart::HandR) {
+
+ for (int w = 0; w < window; w++) {
+ int index = i + w;
+ if (index - 1 >= 0 && index < AnimSaveState.AnimData.Num()) {
+ FProcessedAnimData& Data0 = AnimSaveState.AnimData[index - 1];
+ FProcessedAnimData& Data1 = AnimSaveState.AnimData[index];
+ float Time = (Data1.Timestamp - Data0.Timestamp).GetTotalSeconds();
+ Speed += Time * (Data1.SensorData.GetEntry(Type)->Pos - Data0.SensorData.GetEntry(Type)->Pos).Size();
+
+ }
+ }
+
+ SpeedCount++;
+
+ }
+
+ }
+
+ GestureSpeed.Push(Speed / (float)SpeedCount);
+
+ }
+
+ //smooth
+ TArray<float> SmoothedGestureSpeed;
+ int window = 5;
+ int Num = GestureSpeed.Num();
+ for (int i = 0; i < Num; i++) {
+ float avg = 0.f;
+ for (int wInd = -window; wInd <= window; wInd++) {
+ int w = i + wInd;
+ if (w < 0) {
+ avg += (GestureSpeed[0] + GestureSpeed[1] + GestureSpeed[2] + GestureSpeed[3] + GestureSpeed[4]) / 5.f;
+ }
+ else if (w >= Num) {
+ avg += (GestureSpeed[Num - 1] + GestureSpeed[Num - 2] + GestureSpeed[Num - 3] + GestureSpeed[Num - 4] + GestureSpeed[Num - 5]) / 5.f;
+ }
+ else {
+ avg += GestureSpeed[w];
+ }
+ }
+ avg /= (float)(window * 2 + 1);
+ SmoothedGestureSpeed.Push(avg);
+ }
+
+ GestureSpeed = SmoothedGestureSpeed;
+
+ int MaxNumToScale = 30;
+
+ //maximum
+ int maxWindow = 8;
+ int ScaleStart;
+ for (ScaleStart = GestureSpeed.Num() - 1; ScaleStart >= 0 && ScaleStart >= GestureSpeed.Num() - MaxNumToScale; ScaleStart--) {
+ bool isMax = true;
+ for (int w = -maxWindow; w <= maxWindow; w++) {
+ int index = ScaleStart + w;
+ if (index >= 0 && index < GestureSpeed.Num() && index != ScaleStart) {
+ if (GestureSpeed[index] >= GestureSpeed[ScaleStart]) {
+ isMax = false;
+ break;
+ }
+ }
+ }
+ if (isMax) {
+ break;
+ }
+ }
+
+ ScaleAnimDataInterval(ScaleStart, AnimSaveState.AnimData.Num() - 2, GestureHoldExcessTime);
+
+ }
+
+ AnimSaveState.NextFrame = FTimespan();
+
+
+ //debugging things
+
+ std::string speed = "";
+ std::string lp = "";
+ std::string lpo = "";
+
+ for (int i = 0; i < GestureSpeed.Num(); i++) {
+ speed += "\n" + std::to_string(GestureSpeed[i]);
+ }
+
+ for (int i = 0; i < AnimSaveState.LowestPoints.Num(); i++) {
+ lp += "\n" + std::to_string(AnimSaveState.LowestPoints[i]);
+ }
+
+ for (int i = 0; i < AnimSaveState.LowestPointsOffset.Num(); i++) {
+ lpo += "\n" + std::to_string(AnimSaveState.LowestPointsOffset[i]);
+ }
+
+}
+
+
+void AMCController::InputNextFrame() {
+
+ bool stop = false;
+ UMCAnimInstance* AI = AnimSaveState.Pawn->GetAnimInstance();
+
+ for (; !stop && AnimSaveState.CurrentEntryIndex < AnimSaveState.AnimData.Num();) {
+
+ auto& Entry = AnimSaveState.AnimData[AnimSaveState.CurrentEntryIndex];
+ AnimSaveState.CurrentEntryIndex++;
+
+ if (Entry.IsMarker || Entry.IsEnd) {
if (AnimSaveState.CurrentMarker != 0) {
AI->SnapshotAnimations.Last().Fps = AnimSaveState.FPS;
- AI->SnapshotAnimations.Last().EndTime = Timestamp;
+ AI->SnapshotAnimations.Last().EndTime = Entry.Timestamp;
}
- if (Type == "Marker") {
+ if (Entry.IsMarker) {
AI->SnapshotAnimations.Add(FSnapshotAnimations());
- AI->SnapshotAnimations.Last().StartTime = Timestamp;
+ AI->SnapshotAnimations.Last().StartTime = Entry.Timestamp;
AnimSaveState.CurrentMarker++;
FeedbackWidget->RecordingStateText->SetText(FText::FromString("Converting Anim " + FString::FromInt(AnimSaveState.CurrentMarker) + "..."));
}
else { // if (Type == "End")
+ AnimSaveState.CurrentEntryIndex++;
stop = true;
}
}
- else if (Type == "ViveData") {
- AnimSaveState.Pawn->InputViveDataToAnimInstance(TmpJson);
+
+ else {
+
+ AI->SensorData = Entry.SensorData;
if (bFingerTrackingEnabled) {
- AnimSaveState.Pawn->InputFingerDataToAnimInstance(TmpJson);
+ AI->FingerData = Entry.FingerData;
}
- }
- else if (Type == "Offsets") {
- AnimSaveState.Pawn->InputViveOffsetsToAnimInstance(TmpJson);
- }
- AnimSaveState.CurrentEntryIndex++;
+ AI->SnapshotAnimations.Last().TimeStamps.Push(Entry.Timestamp);
+
+ AnimSaveState.Pawn->AICalcFrame();
+
+ stop = true;
+ AnimSaveState.WaitForAnimInstance = true;
+
+ FeedbackWidget->ProgBar->SetPercent((float)AnimSaveState.CurrentEntryIndex / (float)AnimSaveState.AnimData.Num());
+
+ }
+
}
@@ -503,6 +972,8 @@ void AMCController::SaveAnimation() {
IsSavingToAnim = true;
AnimSaveState.WaitForAnimInstance = true;
+ PreprocessRecording();
+
InputNextFrame();
}
diff --git a/Source/MoCapPlugin/Private/MCDefines.cpp b/Source/MoCapPlugin/Private/MCDefines.cpp
new file mode 100644
index 0000000..df56a0e
--- /dev/null
+++ b/Source/MoCapPlugin/Private/MCDefines.cpp
@@ -0,0 +1,38 @@
+#include "MCDefines.h"
+
+FSensorDataEntry* FSensorData::GetEntry(EBodyPart BodyPart) {
+ switch (BodyPart) {
+ case EBodyPart::Head:
+ return &Head;
+ case EBodyPart::HandL:
+ return &HandL;
+ case EBodyPart::HandR:
+ return &HandR;
+ case EBodyPart::FootL:
+ return &FootL;
+ case EBodyPart::FootR:
+ return &FootR;
+ case EBodyPart::UpperArmL:
+ return &UpperArmL;
+ case EBodyPart::UpperArmR:
+ return &UpperArmR;
+ case EBodyPart::LowerArmL:
+ return &LowerArmL;
+ case EBodyPart::LowerArmR:
+ return &LowerArmR;
+ case EBodyPart::UpperLegL:
+ return &UpperLegL;
+ case EBodyPart::UpperLegR:
+ return &UpperLegR;
+ case EBodyPart::LowerLegL:
+ return &LowerLegL;
+ case EBodyPart::LowerLegR:
+ return &LowerLegR;
+ case EBodyPart::UpperBody:
+ return &UpperBody;
+ case EBodyPart::LowerBody:
+ return &LowerBody;
+ default:
+ return nullptr;
+ }
+}
diff --git a/Source/MoCapPlugin/Private/MCPawn.cpp b/Source/MoCapPlugin/Private/MCPawn.cpp
index 99f019f..b8a6454 100644
--- a/Source/MoCapPlugin/Private/MCPawn.cpp
+++ b/Source/MoCapPlugin/Private/MCPawn.cpp
@@ -27,8 +27,9 @@ AMCPawn::AMCPawn(const FObjectInitializer& ObjectInitializer)
SkeletalMesh->SetVisibility(true, true);
SkeletalMesh->SetupAttachment(RootComponent);
- SensorSetup.Trackers.Add(FSensor(EBodyPart::LowerLegL, "Tracker_Foot_Left", BoneNames.calf_l));
- SensorSetup.Trackers.Add(FSensor(EBodyPart::LowerLegR, "Tracker_Foot_Right", BoneNames.calf_r));
+
+ SensorSetup.Trackers.Add(FSensor(EBodyPart::LowerLegL, "Tracker_Foot_Left", BoneNames.foot_l));
+ SensorSetup.Trackers.Add(FSensor(EBodyPart::LowerLegR, "Tracker_Foot_Right", BoneNames.foot_r));
SensorSetup.Trackers.Add(FSensor(EBodyPart::LowerArmL, "Tracker_Elbow_Left", BoneNames.lowerarm_l));
SensorSetup.Trackers.Add(FSensor(EBodyPart::LowerArmR, "Tracker_Elbow_Right", BoneNames.lowerarm_r));
SensorSetup.Trackers.Add(FSensor(EBodyPart::UpperBody, "Tracker_Chest", BoneNames.spine03));
@@ -413,13 +414,14 @@ void AMCPawn::InputViveDataToAnimInstance(TSharedPtr<FJsonObject> Data) {
const float Roll = FCString::Atof(*JsonObj->GetStringField("Roll"));
const float Pitch = FCString::Atof(*JsonObj->GetStringField("Pitch"));
const float Yaw = FCString::Atof(*JsonObj->GetStringField("Yaw"));
+ FRotator Rot(Pitch, Yaw, Roll);
const float PosX = FCString::Atof(*JsonObj->GetStringField("PosX"));
const float PosY = FCString::Atof(*JsonObj->GetStringField("PosY"));
const float PosZ = FCString::Atof(*JsonObj->GetStringField("PosZ"));
+ FVector Pos(PosX, PosY, PosZ);
- AI->SetSensorData((EBodyPart)i, FVector(PosX, PosY, PosZ), FRotator(Pitch, Yaw, Roll).Quaternion());
-
+ AI->SetSensorData((EBodyPart)i, Pos, Rot.Quaternion());
}
}
@@ -485,6 +487,50 @@ void AMCPawn::InputFingerDataToAnimInstance(TSharedPtr<FJsonObject> Data) {
}
+void AMCPawn::OffsetSensorData(EBodyPart part, FVector & Pos, FQuat & Rot) {
+
+ FSensorOffset* Offset = nullptr;
+ UMCAnimInstance* AI = GetAnimInstance();
+
+ switch (part) {
+ case EBodyPart::Head:
+ Offset = &AI->SensorOffsets.Head;
+ break;
+ case EBodyPart::HandL:
+ Offset = &AI->SensorOffsets.HandL;
+ break;
+ case EBodyPart::HandR:
+ Offset = &AI->SensorOffsets.HandR;
+ break;
+ case EBodyPart::LowerLegL:
+ case EBodyPart::FootL:
+ Offset = &AI->SensorOffsets.LowerLegL;
+ break;
+ case EBodyPart::LowerLegR:
+ case EBodyPart::FootR:
+ Offset = &AI->SensorOffsets.LowerLegR;
+ break;
+ case EBodyPart::LowerArmL:
+ Offset = &AI->SensorOffsets.LowerArmL;
+ break;
+ case EBodyPart::LowerArmR:
+ Offset = &AI->SensorOffsets.LowerArmR;
+ break;
+ case EBodyPart::LowerBody:
+ Offset = &AI->SensorOffsets.LowerBody;
+ break;
+ case EBodyPart::UpperBody:
+ Offset = &AI->SensorOffsets.UpperBody;
+ break;
+ }
+
+ FQuat RotTmp = Rot;
+
+ Rot = Rot * Offset->Rot;
+ Pos = Pos - Offset->Distance * (Offset->AxisRotDiff * RotTmp).GetAxisX();
+
+}
+
void AMCPawn::AddSensorDataToJson(TSharedPtr<FJsonObject> JsonObjectFull, const FSensor& Sensor) {
TSharedPtr<FJsonObject> JsonObject = MAKE_JSON;
diff --git a/Source/MoCapPlugin/Private/MCRigUnits.cpp b/Source/MoCapPlugin/Private/MCRigUnits.cpp
index 72cfa98..950e55b 100644
--- a/Source/MoCapPlugin/Private/MCRigUnits.cpp
+++ b/Source/MoCapPlugin/Private/MCRigUnits.cpp
@@ -154,21 +154,74 @@ FRigUnit_ApplyFingerData_Execute() {
}
}
-FRigUnit_CheckLengths::FRigUnit_CheckLengths() {
+FRigUnit_FootLocking::FRigUnit_FootLocking() {
+
+ FloorOffset = 0.f;
+ ReleaseHeight = 0.2f;
+ ReleaseDistHorizontal = 0.3f;
+
+ LockedLeft = false;
+ LockedRight = false;
}
-FRigUnit_CheckLengths_Execute() {
+FRigUnit_FootLocking_Execute() {
DECLARE_SCOPE_HIERARCHICAL_COUNTER_RIGUNIT()
- FRigBoneHierarchy* Hierarchy = ExecuteContext.GetBones();
+ FRigBoneHierarchy* Bones = ExecuteContext.GetBones();
+ FRigControlHierarchy* Controls = ExecuteContext.GetControls();
- if (Hierarchy) {
+ if (Bones && Controls) {
+
+ FName FootBoneName;
+ bool* Locked = nullptr;
+ FTransform* LockedTrans = nullptr;
+
+ for (int i = 0; i < 2; i++) {
+
+ if (i == 0) {
+ FootBoneName = "CalfL";
+ Locked = &LockedLeft;
+ LockedTrans = &LockedTransLeft;
+ }
+ else {
+ FootBoneName = "CalfR";
+ Locked = &LockedRight;
+ LockedTrans = &LockedTransRight;
+ }
+
+ FTransform Root = Bones->GetGlobalTransform("root");
+ FTransform Foot = Controls->GetGlobalTransform(FootBoneName);
- FTransform Trans1 = Hierarchy->GetGlobalTransform("pelvis");
- FTransform Trans2 = Hierarchy->GetGlobalTransform("neck_01");
+ FVector RootTrans2D = Root.GetLocation();
+ float RootHeight = RootTrans2D.Z;
+ RootTrans2D.Z = 0.f;
+ FVector FootTrans2D = Root.GetLocation();
+ float FootHeight = FootTrans2D.Z;
+ FootTrans2D.Z = 0.f;
- float Dist = FVector::Dist(Trans1.GetLocation(), Trans2.GetLocation());
+ if (*Locked) {
+
+ float Dist2D = FVector::Dist(RootTrans2D, FootTrans2D);
+
+ if (FootHeight > RootHeight + FloorOffset + ReleaseHeight || Dist2D > ReleaseDistHorizontal) {
+ *Locked = false;
+ }
+ else {
+ Controls->SetGlobalTransform(FootBoneName, *LockedTrans);
+ }
+
+ }
+ else {
+
+ if (FootHeight < RootHeight + FloorOffset) {
+ *Locked = true;
+ *LockedTrans = Foot;
+ }
+
+ }
+
+ }
}
}
\ No newline at end of file
diff --git a/Source/MoCapPlugin/Public/MCController.h b/Source/MoCapPlugin/Public/MCController.h
index 63692fc..0f4dab3 100644
--- a/Source/MoCapPlugin/Public/MCController.h
+++ b/Source/MoCapPlugin/Public/MCController.h
@@ -16,14 +16,23 @@
#include "MCController.generated.h"
-class USerial;
-class UPoseAsset;
+struct FProcessedAnimData {
+ bool IsMarker = false;
+ bool IsEnd = false;
+ FTimespan Timestamp;
+ FFingerData FingerData;
+ FSensorData SensorData;
+};
USTRUCT()
struct FAnimSaveState {
GENERATED_BODY()
+ TArray<float> LowestPoints;
+ TArray<float> LowestPointsOffset;
+
TArray<FString> StringData;
+ TArray<FProcessedAnimData> AnimData;
int CurrentEntryIndex;
int CurrentMarker;
FTimespan LastMarker;
@@ -79,6 +88,8 @@ protected:
void RecordMode();
void SaveToAnimMode();
+ void ScaleAnimDataInterval(int start, int end, float DeltaTime);
+ void PreprocessRecording();
void InputNextFrame();
void SaveAnimSnapshots();
@@ -128,4 +139,7 @@ public:
UPROPERTY(EditAnywhere, meta = (DisplayName = "Spectator Cam", Category = "MotionCapture"))
ASceneCapture2D* SpectatorCam;
+ UPROPERTY(EditAnywhere, meta = (DisplayName = "Gesture Hold Scale Excess Time", Category = "MotionCapture"))
+ float GestureHoldExcessTime = 0.f;
+
};
diff --git a/Source/MoCapPlugin/Public/MCDefines.h b/Source/MoCapPlugin/Public/MCDefines.h
index ae4ed36..335a5ab 100644
--- a/Source/MoCapPlugin/Public/MCDefines.h
+++ b/Source/MoCapPlugin/Public/MCDefines.h
@@ -96,6 +96,8 @@ struct FSnapshotAnimations {
UPROPERTY(BlueprintReadWrite)
TArray<FPoseSnapshot> Snapshots;
+ TArray<FTimespan> TimeStamps;
+
int Fps;
FTimespan StartTime;
FTimespan EndTime;
@@ -169,6 +171,8 @@ struct FSensorData {
UPROPERTY(BlueprintReadOnly)
FSensorDataEntry LowerBody;
+ FSensorDataEntry* GetEntry(EBodyPart BodyPart);
+
};
USTRUCT(BlueprintType)
diff --git a/Source/MoCapPlugin/Public/MCPawn.h b/Source/MoCapPlugin/Public/MCPawn.h
index ea0ae7b..47acc9b 100644
--- a/Source/MoCapPlugin/Public/MCPawn.h
+++ b/Source/MoCapPlugin/Public/MCPawn.h
@@ -26,6 +26,8 @@ class MOCAPPLUGIN_API AMCPawn : public APawn
public:
+ bool UseKneeAsFoot;
+
UPROPERTY(BlueprintReadOnly)
FSensorSetup SensorSetup;
@@ -64,6 +66,8 @@ public:
void InputViveOffsetsToAnimInstance(TSharedPtr<FJsonObject> Data);
void InputFingerDataToAnimInstance(TSharedPtr<FJsonObject> Data);
+ void OffsetSensorData(EBodyPart part, FVector& Pos, FQuat& Rot);
+
private:
void AddSensorDataToJson(TSharedPtr<FJsonObject> JsonObjectFull, const FSensor& Sensor);
diff --git a/Source/MoCapPlugin/Public/MCRigUnits.h b/Source/MoCapPlugin/Public/MCRigUnits.h
index e1edeac..a6668e4 100644
--- a/Source/MoCapPlugin/Public/MCRigUnits.h
+++ b/Source/MoCapPlugin/Public/MCRigUnits.h
@@ -73,14 +73,35 @@ struct FRigUnit_ApplyFingerData : public FRigUnitMutable
};
-USTRUCT(meta = (DisplayName = "CheckLengths", PrototypeName = "CheckLengths", Category = "Hierarchy", NodeColor = "0.05 0.25 0.05"))
-struct FRigUnit_CheckLengths : public FRigUnitMutable
+USTRUCT(meta = (DisplayName = "FootLocking", PrototypeName = "FootLocking", Category = "Hierarchy", NodeColor = "0.05 0.25 0.05"))
+struct FRigUnit_FootLocking : public FRigUnitMutable
{
GENERATED_BODY()
- FRigUnit_CheckLengths();
+ FRigUnit_FootLocking();
RIGVM_METHOD()
virtual void Execute(const FRigUnitContext& Context) override;
+ UPROPERTY(meta = (Input))
+ float FloorOffset;
+
+ UPROPERTY(meta = (Input))
+ float ReleaseHeight;
+
+ UPROPERTY(meta = (Input))
+ float ReleaseDistHorizontal;
+
+ UPROPERTY()
+ bool LockedLeft;
+
+ UPROPERTY()
+ bool LockedRight;
+
+ UPROPERTY()
+ FTransform LockedTransLeft;
+
+ UPROPERTY()
+ FTransform LockedTransRight;
+
};
\ No newline at end of file
--
GitLab