Added interpolation system

include/libc/math.h

@@ -48,7 +48,13 @@ typedef union {
 #define __lnearbyintf lnearbyintf
 #define __lnearbyint lnearbyint
 
-extern f32 __sinf(f32);
+#ifdef __cplusplus
-extern f32 __cosf(f32);
+extern "C" {
+#endif
+f32 __sinf(f32);
+f32 __cosf(f32);
+#ifdef __cplusplus
+}
+#endif
 
 #endif

include/sf64math.h

@@ -55,6 +55,10 @@ typedef union {
     // u64 force_struct_alignment;
 } Matrix; // size = 0x40
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 extern Mtx gIdentityMtx; // 800C4620
 extern Matrix gIdentityMatrix; //800C4660
 
@@ -93,6 +97,8 @@ void Matrix_Pop(Matrix** mtxStack);
 // Copies tf into mtx (MTXF_NEW) or applies it to mtx (MTXF_APPLY)
 void Matrix_Mult(Matrix* mtx, Matrix* tf, u8 mode);
 
+void Matrix_MtxFMtxFMult(MtxF* mfB, MtxF* mfA, MtxF* dest);
+
 // Creates a translation matrix in mtx (MTXF_NEW) or applies one to mtx (MTXF_APPLY)
 void Matrix_Translate(Matrix* mtx, f32 x, f32 y, f32 z, u8 mode);
 
@@ -155,4 +161,8 @@ f32 Math_NearbyIntF(f32);
 f32 Math_TruncF(f32);
 f32 Math_RoundF(f32);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

src/engine/fox_beam.c

@@ -3,6 +3,7 @@
 #include "assets/ast_aquas.h"
 #include "assets/ast_great_fox.h"
 #include "assets/ast_versus.h"
+#include "port/interpolation/FrameInterpolation.h"
 
 Vec3f sShotViewPos;
 
@@ -1502,6 +1503,7 @@ void PlayerShot_DrawShot(PlayerShot* shot) {
 }
 
 void PlayerShot_Draw(PlayerShot* shot) {
+    FrameInterpolation_RecordOpenChild(shot, 0);
     switch (shot->obj.status) {
         case SHOT_ACTIVE:
             PlayerShot_DrawShot(shot);
@@ -1510,6 +1512,7 @@ void PlayerShot_Draw(PlayerShot* shot) {
             PlayerShot_DrawHitmark(shot);
             break;
     }
+    FrameInterpolation_RecordCloseChild();
 }
 
 void PlayerShot_UpdateHitmark(PlayerShot* shot) {

src/engine/fox_bg.c

@@ -22,6 +22,7 @@
 #include "assets/ast_zoness.h"
 
 #include "prevent_bss_reordering2.h"
+#include "port/interpolation/FrameInterpolation.h"
 // #include "prevent_bss_reordering3.h"
 
 f32 gWarpZoneBgAlpha;
@@ -252,6 +253,7 @@ void Background_DrawBackdrop(void) {
     u8 levelType;
     s32 levelId;
 
+
     if (gDrawBackdrop == 0) {
         return;
     }

src/engine/fox_display.c

@@ -4,6 +4,7 @@
 #include "assets/ast_landmaster.h"
 #include "assets/ast_versus.h"
 #include "assets/ast_sector_z.h"
+#include "port/interpolation/FrameInterpolation.h"
 
 Vec3f D_display_801613B0[4];
 Vec3f D_display_801613E0[4];
@@ -710,6 +711,7 @@ void Display_Reticle(Player* player) {
         (((gGameState == GSTATE_PLAY) && (player->state_1C8 == PLAYERSTATE_1C8_ACTIVE)) ||
          (gGameState == GSTATE_MENU))) {
         for (i = 0; i < 2; i++) {
+            FrameInterpolation_RecordOpenChild("Reticle", i);
             translate = &D_display_801613E0[i];
             Matrix_Push(&gGfxMatrix);
             Matrix_Translate(gGfxMatrix, translate->x, translate->y, translate->z, MTXF_APPLY);
@@ -735,6 +737,7 @@ void Display_Reticle(Player* player) {
             Matrix_SetGfxMtx(&gMasterDisp);
             gSPDisplayList(gMasterDisp++, D_1024F60);
             Matrix_Pop(&gGfxMatrix);
+            FrameInterpolation_RecordCloseChild();
         }
     }
 }
@@ -1502,6 +1505,7 @@ void Display_ActorMarks(void) {
         RCP_SetupDL_40();
 
         for (i = 0; i < ARRAY_COUNT(gTeamArrowsViewPos); i++) {
+            FrameInterpolation_RecordOpenChild(&gTeamArrowsViewPos[i],  i);
             if (gTeamArrowsViewPos[i].z < 0.0f) {
                 var_fs0 = (VEC3F_MAG(&gTeamArrowsViewPos[i])) * 0.0015f;
                 if (var_fs0 > 100.0f) {
@@ -1532,6 +1536,7 @@ void Display_ActorMarks(void) {
             }
             gTeamArrowsViewPos[i].x = gTeamArrowsViewPos[i].y = 0;
             gTeamArrowsViewPos[i].z = 100.0f;
+            FrameInterpolation_RecordCloseChild();
         }
         gDPSetTextureFilter(gMasterDisp++, G_TF_BILERP);
     }
@@ -1742,8 +1747,12 @@ void Display_Update(void) {
         Background_DrawStarfield();
     }
 
+    FrameInterpolation_RecordOpenChild("Backdrop", 0);
     Background_DrawBackdrop();
+    FrameInterpolation_RecordCloseChild();
+    FrameInterpolation_RecordOpenChild("Sun", 0);
     Background_DrawSun();
+    FrameInterpolation_RecordCloseChild();
     Matrix_Push(&gGfxMatrix);
     Matrix_LookAt(gGfxMatrix, gPlayCamEye.x, gPlayCamEye.y, gPlayCamEye.z, gPlayCamAt.x, gPlayCamAt.y, gPlayCamAt.z,
                   playerCamUp.x, playerCamUp.y, playerCamUp.z, MTXF_APPLY);
@@ -1759,7 +1768,9 @@ void Display_Update(void) {
             Matrix_Pop(&gGfxMatrix);
         } else if (gGroundSurface != SURFACE_WATER) {
             D_bg_8015F964 = false;
+            FrameInterpolation_RecordOpenChild("Ground", 0);
             Background_DrawGround();
+            FrameInterpolation_RecordCloseChild();
         }
     }
 
@@ -1823,7 +1834,9 @@ void Display_Update(void) {
     if ((gGroundSurface == SURFACE_WATER) || (gAqDrawMode != 0)) {
         D_bg_8015F964 = true;
         Effect_Draw(1);
+        FrameInterpolation_RecordOpenChild("Ground", 0);
         Background_DrawGround();
+        FrameInterpolation_RecordCloseChild();
     }
 
     if ((gCurrentLevel != LEVEL_AQUAS) &&
@@ -1837,9 +1850,11 @@ void Display_Update(void) {
 
     for (i = 0, player = &gPlayer[0]; i < gCamCount; i++, player++) {
         if (sPlayersVisible[i]) {
+            FrameInterpolation_RecordOpenChild(player, i);
             Display_PlayerShadow_Update(player);
             Display_PlayerFeatures(player);
             Display_ArwingWingTrail_Update(player);
+            FrameInterpolation_RecordCloseChild();
         }
     }
 

src/engine/fox_edisplay.c

@@ -22,6 +22,7 @@
 #include "assets/ast_enmy_planet.h"
 #include "assets/ast_ve1_boss.h"
 #include "assets/ast_zoness.h"
+#include "port/interpolation/FrameInterpolation.h"
 
 Vec3f D_edisplay_801615D0;
 Vec3f sViewPos;
@@ -1674,6 +1675,7 @@ void Object_DrawAll(s32 arg0) {
 
         for (i = 0, scenery360 = gScenery360; i < 200; i++, scenery360++) {
             if ((scenery360->obj.status == OBJ_ACTIVE) && (scenery360->obj.id != OBJ_SCENERY_LEVEL_OBJECTS)) {
+                FrameInterpolation_RecordOpenChild(scenery360, i);
                 if (gCurrentLevel == LEVEL_BOLSE) {
                     spAC.x = scenery360->sfxSource[0];
                     spAC.y = scenery360->sfxSource[1];
@@ -1684,12 +1686,14 @@ void Object_DrawAll(s32 arg0) {
                 Matrix_Push(&gGfxMatrix);
                 Scenery360_Draw(scenery360);
                 Matrix_Pop(&gGfxMatrix);
+                FrameInterpolation_RecordCloseChild();
             }
         }
     } else {
         RCP_SetupDL_29(gFogRed, gFogGreen, gFogBlue, gFogAlpha, gFogNear, gFogFar);
         for (i = 0, scenery = gScenery; i < ARRAY_COUNT(gScenery); i++, scenery++) {
             if (scenery->obj.status >= OBJ_ACTIVE) {
+                FrameInterpolation_RecordOpenChild(scenery, i);
                 if (arg0 > 0) {
                     Display_SetSecondLight(&scenery->obj.pos);
                 }
@@ -1697,12 +1701,14 @@ void Object_DrawAll(s32 arg0) {
                 Scenery_Draw(scenery, arg0);
                 Matrix_Pop(&gGfxMatrix);
                 Object_UpdateSfxSource(scenery->sfxSource);
+                FrameInterpolation_RecordCloseChild();
             }
         }
     }
 
     for (i = 0, boss = gBosses; i < ARRAY_COUNT(gBosses); i++, boss++) {
         if ((boss->obj.status >= OBJ_ACTIVE) && (boss->obj.id != OBJ_BOSS_BO_BASE_SHIELD)) {
+            FrameInterpolation_RecordOpenChild(boss, i);
             if ((boss->timer_05C % 2) == 0) {
                 RCP_SetupDL_29(gFogRed, gFogGreen, gFogBlue, gFogAlpha, gFogNear, gFogFar);
             } else {
@@ -1718,6 +1724,7 @@ void Object_DrawAll(s32 arg0) {
                 Object_DrawShadow(i, &boss->obj);
                 Matrix_Pop(&gGfxMatrix);
             }
+            FrameInterpolation_RecordCloseChild();
         }
     }
 
@@ -1726,6 +1733,7 @@ void Object_DrawAll(s32 arg0) {
 
     for (i = 0, sprite = gSprites; i < ARRAY_COUNT(gSprites); i++, sprite++) {
         if ((sprite->obj.status >= OBJ_ACTIVE) && func_enmy_80060FE4(&sprite->obj.pos, -12000.0f)) {
+            FrameInterpolation_RecordOpenChild(sprite, i);
             Matrix_Push(&gGfxMatrix);
 
             if ((sprite->obj.id == OBJ_SPRITE_CO_RUIN1) || (sprite->obj.id == OBJ_SPRITE_CO_RUIN2)) {
@@ -1736,11 +1744,13 @@ void Object_DrawAll(s32 arg0) {
 
             Sprite_Draw(sprite, arg0);
             Matrix_Pop(&gGfxMatrix);
+            FrameInterpolation_RecordCloseChild();
         }
     }
 
     for (i = 0, actor = &gActors[0]; i < ARRAY_COUNT(gActors); i++, actor++) {
         if (actor->obj.status >= OBJ_ACTIVE) {
+            FrameInterpolation_RecordOpenChild(actor, i);
             if ((actor->timer_0C6 % 2) == 0) {
                 if (gCurrentLevel == LEVEL_UNK_15) {
                     RCP_SetupDL_23();
@@ -1789,6 +1799,7 @@ void Object_DrawAll(s32 arg0) {
                     }
                     break;
             }
+        FrameInterpolation_RecordCloseChild();  
         }
     }
 
@@ -1799,11 +1810,13 @@ void Object_DrawAll(s32 arg0) {
 
     for (i = 0, item = &gItems[0]; i < ARRAY_COUNT(gItems); i++, item++) {
         if (item->obj.status >= OBJ_ACTIVE) {
+            FrameInterpolation_RecordOpenChild(item, i);
             Matrix_Push(&gGfxMatrix);
             RCP_SetupDL(&gMasterDisp, SETUPDL_29);
             Object_SetCullDirection(arg0);
             Item_Draw(item, arg0);
             Matrix_Pop(&gGfxMatrix);
+            FrameInterpolation_RecordCloseChild();
         }
     }
 
@@ -1820,6 +1833,7 @@ void Effect_DrawAll(s32 arg0) {
 
     for (i = 0, effect = &gEffects[0]; i < ARRAY_COUNT(gEffects); i++, effect++) {
         if (effect->obj.status >= OBJ_ACTIVE) {
+            FrameInterpolation_RecordOpenChild(effect, i);
             if (effect->info.unk_14 == 1) {
                 effect->obj.rot.y = RAD_TO_DEG(-gPlayer[gPlayerNum].camYaw);
                 effect->obj.rot.x = RAD_TO_DEG(gPlayer[gPlayerNum].camPitch);
@@ -1840,11 +1854,13 @@ void Effect_DrawAll(s32 arg0) {
                     Matrix_Pop(&gGfxMatrix);
                 }
             }
+            FrameInterpolation_RecordCloseChild();
         }
     }
 
     for (i = 0, boss = &gBosses[0]; i < ARRAY_COUNT(gBosses); i++, boss++) {
         if ((boss->obj.status >= OBJ_ACTIVE) && (boss->obj.id == OBJ_BOSS_BO_BASE_SHIELD)) {
+            FrameInterpolation_RecordOpenChild(boss, i);
             if ((boss->timer_05C % 2) == 0) {
                 RCP_SetupDL_29(gFogRed, gFogGreen, gFogBlue, gFogAlpha, gFogNear, gFogFar);
             } else {
@@ -1854,6 +1870,7 @@ void Effect_DrawAll(s32 arg0) {
             Matrix_Push(&gGfxMatrix);
             Boss_Draw(boss, arg0);
             Matrix_Pop(&gGfxMatrix);
+            FrameInterpolation_RecordCloseChild();
         }
     }
 }
@@ -1914,6 +1931,7 @@ void TexturedLine_Draw(void) {
 
     for (i = 0; i < ARRAY_COUNT(gTexturedLines); i++) {
         TexturedLine* texLine = &gTexturedLines[i];
+        FrameInterpolation_RecordOpenChild(texLine, i);
 
         if (gTexturedLines[i].mode != 0) {
             Matrix_Push(&gGfxMatrix);
@@ -1953,6 +1971,7 @@ void TexturedLine_Draw(void) {
             }
             Matrix_Pop(&gGfxMatrix);
         }
+        FrameInterpolation_RecordCloseChild();
     }
 }
 

src/engine/fox_game.c

@@ -3,6 +3,7 @@
 #include "sf64dma.h"
 #include "assets/ast_logo.h"
 #include "mods.h"
+#include "port/interpolation/FrameInterpolation.h"
 
 f32 gNextVsViewScale;
 f32 gVsViewScale;
@@ -267,6 +268,7 @@ void Game_InitViewport(Gfx** dList, u8 camCount, u8 camIndex) {
 }
 
 void Game_Draw(s32 playerNum) {
+    FrameInterpolation_StartRecord();
     switch (gDrawMode) {
         case DRAW_NONE:
             break;
@@ -303,6 +305,7 @@ void Game_Draw(s32 playerNum) {
             Ending_Draw();
             break;
     }
+    FrameInterpolation_StopRecord();
 }
 
 void Game_SetScene(void) {

src/engine/fox_hud.c

@@ -1,6 +1,7 @@
 #include "sf64math.h"
 #include "fox_hud.h"
 #include "prevent_bss_reordering.h"
+#include "port/interpolation/FrameInterpolation.h"
 
 Vec3f D_801616A0;
 Vec3f D_801616B0;

src/port/Engine.cpp

@@ -17,6 +17,7 @@
 #include "resource/importers/SkeletonFactory.h"
 #include "resource/importers/Vec3fFactory.h"
 #include "resource/importers/Vec3sFactory.h"
+#include "port/interpolation/FrameInterpolation.h"
 #include <Fast3D/Fast3dWindow.h>
 #include <DisplayListFactory.h>
 #include <TextureFactory.h>
@@ -113,9 +114,11 @@ void GameEngine::StartFrame() const{
     this->context->GetWindow()->StartFrame();
 }
 
-void GameEngine::RunCommands(Gfx* Commands) {
+void GameEngine::RunCommands(Gfx* Commands, const std::vector<std::unordered_map<Mtx*, MtxF>>& mtx_replacements) {
-    gfx_run(Commands, {});
+    for (const auto& m : mtx_replacements) {
+        gfx_run(Commands, m);
         gfx_end_frame();
+    }
 
     if (ShouldClearTextureCacheAtEndOfFrame) {
         gfx_texture_cache_clear();
@@ -133,10 +136,67 @@ void GameEngine::ProcessGfxCommands(Gfx* commands) {
     gFPS = 30;
     wnd->EnableSRGBMode();
     wnd->SetRendererUCode(UcodeHandlers::ucode_f3dex);
-    wnd->SetTargetFps(60 / gVIsPerFrame);
-    wnd->SetMaximumFrameLatency(1);
 
-    RunCommands(commands);
+    std::vector<std::unordered_map<Mtx*, MtxF>> mtx_replacements;
+    int target_fps = CVarGetInteger("gInterpolationFPS", 20);
+    static int last_fps;
+    static int last_update_rate;
+    static int time;
+    int fps = target_fps;
+    int original_fps = gFPS = 60 / gVIsPerFrame;
+
+    if (target_fps == 20 || original_fps > target_fps) {
+        fps = original_fps;
+    }
+
+    if (last_fps != fps || last_update_rate != gVIsPerFrame) {
+        time = 0;
+    }
+
+    // time_base = fps * original_fps (one second)
+    int next_original_frame = fps;
+
+    while (time + original_fps <= next_original_frame) {
+        time += original_fps;
+        if (time != next_original_frame) {
+            mtx_replacements.push_back(FrameInterpolation_Interpolate((float)time / next_original_frame));
+        } else {
+            mtx_replacements.emplace_back();
+        }
+    }
+
+    time -= fps;
+
+    int threshold = CVarGetInteger("gExtraLatencyThreshold", 80);
+
+    if (wnd != nullptr) {
+        wnd->SetTargetFps(fps);
+        wnd->SetMaximumFrameLatency(threshold > 0 && target_fps >= threshold ? 2 : 1);
+    }
+
+    // When the gfx debugger is active, only run with the final mtx
+    if (GfxDebuggerIsDebugging()) {
+        mtx_replacements.clear();
+        mtx_replacements.emplace_back();
+    }
+
+    RunCommands(commands, mtx_replacements);
+
+    last_fps = fps;
+    last_update_rate = gVIsPerFrame;
+}
+
+uint32_t GameEngine::GetInterpolationFPS(){
+    if (Ship::Context::GetInstance()->GetWindow()->GetWindowBackend() == Ship::WindowBackend::FAST3D_DXGI_DX11) {
+        return CVarGetInteger("gInterpolationFPS", 20);
+    }
+
+    if (CVarGetInteger("gMatchRefreshRate", 0)) {
+        return Ship::Context::GetInstance()->GetWindow()->GetCurrentRefreshRate();
+    }
+
+    return std::min<uint32_t>(Ship::Context::GetInstance()->GetWindow()->GetCurrentRefreshRate(),
+                              CVarGetInteger("gInterpolationFPS", 20));
 }
 
 extern "C" uint32_t GameEngine_GetSampleRate() {

src/port/Engine.h

@@ -23,10 +23,11 @@ class GameEngine {
     GameEngine();
     static void Create();
     void StartFrame() const;
-    static void RunCommands(Gfx* Commands);
+    static void RunCommands(Gfx* Commands, const std::vector<std::unordered_map<Mtx*, MtxF>>& mtx_replacements);
     void ProcessFrame(void (*run_one_game_iter)()) const;
     static void Destroy();
     static void ProcessGfxCommands(Gfx* commands);
+    static uint32_t GetInterpolationFPS();
 };
 #else
 void GameEngine_ProcessGfxCommands(Gfx* commands);

src/port/interpolation/FrameInterpolation.cpp

@@ -0,0 +1,612 @@
+#include <libultraship/bridge.h>
+
+#include <vector>
+#include <map>
+#include <unordered_map>
+#include <math.h>
+#include "port/Engine.h"
+
+#include "FrameInterpolation.h"
+
+/*
+Frame interpolation.
+
+The idea of this code is to interpolate all matrices.
+
+The code contains two approaches. The first is to interpolate
+all inputs in transformations, such as angles, scale and distances,
+and then perform the same transformations with the interpolated values.
+After evaluation for some reason some animations such rolling look strange.
+
+The second approach is to simply interpolate the final matrices. This will
+more or less simply interpolate the world coordinates for movements.
+This will however make rotations ~180 degrees get the "paper effect".
+The mitigation is to identify this case for actors and interpolate the
+matrix but in model coordinates instead, by "removing" the rotation-
+translation before interpolating, create a rotation matrix with the
+interpolated angle which is then applied to the matrix.
+
+Currently the code contains both methods but only the second one is currently
+used.
+
+Both approaches build a tree of instructions, containing matrices
+at leaves. Every node is built from OPEN_DISPS/CLOSE_DISPS and manually
+inserted FrameInterpolation_OpenChild/FrameInterpolation_Close child calls.
+These nodes contain information that should suffice to identify the matrix,
+so we can find it in an adjacent frame.
+
+We can interpolate an arbitrary amount of frames between two original frames,
+given a specific interpolation factor (0=old frame, 0.5=average of frames,
+1.0=new frame).
+*/
+
+static bool invert_matrix(const float m[16], float invOut[16]);
+
+using namespace std;
+
+namespace {
+
+enum class Op {
+    OpenChild,
+    CloseChild,
+
+    MatrixPush,
+    MatrixPop,
+    MatrixPut,
+    MatrixMult,
+    MatrixTranslate,
+    MatrixScale,
+    MatrixRotate1Coord,
+    MatrixMultVec3fNoTranslate,
+    MatrixMultVec3f,
+    MatrixMtxFToMtx,
+    MatrixToMtx,
+    MatrixRotateAxis,
+    SkinMatrixMtxFToMtx
+};
+
+typedef pair<const void*, int> label;
+
+union Data {
+    Data() {
+    }
+
+    struct {
+        MtxF src;
+    } matrix_put;
+
+    struct {
+        MtxF mf;
+        u8 mode;
+    } matrix_mult;
+
+    struct {
+        f32 x, y, z;
+        u8 mode;
+    } matrix_translate, matrix_scale;
+
+    struct {
+        u32 coord;
+        f32 value;
+        u8 mode;
+    } matrix_rotate_1_coord;
+
+    struct {
+        Vec3f src;
+        Vec3f dest;
+    } matrix_vec_translate;
+
+    struct {
+        Vec3f src;
+        Vec3f dest;
+    } matrix_vec_no_translate;
+
+    struct {
+        Vec3f translation;
+        Vec3s rotation;
+    } matrix_translate_rotate_zyx;
+
+    struct {
+        f32 translateX, translateY, translateZ;
+        Vec3s rot;
+        // MtxF mtx;
+        bool has_mtx;
+    } matrix_set_translate_rotate_yxz;
+
+    struct {
+        MtxF src;
+        Mtx* dest;
+    } matrix_mtxf_to_mtx;
+
+    struct {
+        Mtx* dest;
+        MtxF src;
+        bool has_adjusted;
+    } matrix_to_mtx;
+
+    struct {
+        MtxF mf;
+    } matrix_replace_rotation;
+
+    struct {
+        f32 angle;
+        Vec3f axis;
+        u8 mode;
+    } matrix_rotate_axis;
+
+    struct {
+        label key;
+        size_t idx;
+    } open_child;
+};
+
+struct Path {
+    map<label, vector<Path>> children;
+    map<Op, vector<Data>> ops;
+    vector<pair<Op, size_t>> items;
+};
+
+struct Recording {
+    Path root_path;
+};
+
+bool is_recording;
+vector<Path*> current_path;
+uint32_t camera_epoch;
+uint32_t previous_camera_epoch;
+Recording current_recording;
+Recording previous_recording;
+
+bool next_is_actor_pos_rot_matrix;
+bool has_inv_actor_mtx;
+MtxF inv_actor_mtx;
+size_t inv_actor_mtx_path_index;
+
+Data& append(Op op) {
+    auto& m = current_path.back()->ops[op];
+    current_path.back()->items.emplace_back(op, m.size());
+    return m.emplace_back();
+}
+
+MtxF* Matrix_GetCurrent(){
+    return (MtxF*) gGfxMatrix;
+}
+
+struct InterpolateCtx {
+    float step;
+    float w;
+    unordered_map<Mtx*, MtxF> mtx_replacements;
+    MtxF tmp_mtxf, tmp_mtxf2;
+    Vec3f tmp_vec3f, tmp_vec3f2;
+    Vec3s tmp_vec3s;
+    MtxF actor_mtx;
+
+    MtxF* new_replacement(Mtx* addr) {
+        return &mtx_replacements[addr];
+    }
+
+    void interpolate_mtxf(MtxF* res, MtxF* o, MtxF* n) {
+        for (size_t i = 0; i < 4; i++) {
+            for (size_t j = 0; j < 4; j++) {
+                res->mf[i][j] = w * o->mf[i][j] + step * n->mf[i][j];
+            }
+        }
+    }
+
+    float lerp(f32 o, f32 n) {
+        return w * o + step * n;
+    }
+
+    void lerp_vec3f(Vec3f* res, Vec3f* o, Vec3f* n) {
+        res->x = lerp(o->x, n->x);
+        res->y = lerp(o->y, n->y);
+        res->z = lerp(o->z, n->z);
+    }
+
+    float interpolate_angle(f32 o, f32 n) {
+        if (o == n)
+            return n;
+        o = fmodf(o, 2 * M_PI);
+        if (o < 0.0f) {
+            o += 2 * M_PI;
+        }
+        n = fmodf(n, 2 * M_PI);
+        if (n < 0.0f) {
+            n += 2 * M_PI;
+        }
+        if (fabsf(o - n) > M_PI) {
+            if (o < n) {
+                o += 2 * M_PI;
+            } else {
+                n += 2 * M_PI;
+            }
+        }
+        if (fabsf(o - n) > M_PI / 2) {
+            // return n;
+        }
+        return lerp(o, n);
+    }
+
+    s16 interpolate_angle(s16 os, s16 ns) {
+        if (os == ns)
+            return ns;
+        int o = (u16)os;
+        int n = (u16)ns;
+        u16 res;
+        int diff = o - n;
+        if (-0x8000 <= diff && diff <= 0x8000) {
+            if (diff < -0x4000 || diff > 0x4000) {
+                return ns;
+            }
+            res = (u16)(w * o + step * n);
+        } else {
+            if (o < n) {
+                o += 0x10000;
+            } else {
+                n += 0x10000;
+            }
+            diff = o - n;
+            if (diff < -0x4000 || diff > 0x4000) {
+                return ns;
+            }
+            res = (u16)(w * o + step * n);
+        }
+        if (os / 327 == ns / 327 && (s16)res / 327 != os / 327) {
+            int bp = 0;
+        }
+        return res;
+    }
+
+    void interpolate_vecs(Vec3f* res, Vec3f* o, Vec3f* n) {
+        res->x = interpolate_angle(o->x, n->x);
+        res->y = interpolate_angle(o->y, n->y);
+        res->z = interpolate_angle(o->z, n->z);
+    }
+
+    void interpolate_angles(Vec3s* res, Vec3s* o, Vec3s* n) {
+        res->x = interpolate_angle(o->x, n->x);
+        res->y = interpolate_angle(o->y, n->y);
+        res->z = interpolate_angle(o->z, n->z);
+    }
+
+    void interpolate_branch(Path* old_path, Path* new_path) {
+        for (auto& item : new_path->items) {
+            Data& new_op = new_path->ops[item.first][item.second];
+
+            if (item.first == Op::OpenChild) {
+                if (auto it = old_path->children.find(new_op.open_child.key);
+                    it != old_path->children.end() && new_op.open_child.idx < it->second.size()) {
+                    interpolate_branch(&it->second[new_op.open_child.idx],
+                                       &new_path->children.find(new_op.open_child.key)->second[new_op.open_child.idx]);
+                } else {
+                    interpolate_branch(&new_path->children.find(new_op.open_child.key)->second[new_op.open_child.idx],
+                                       &new_path->children.find(new_op.open_child.key)->second[new_op.open_child.idx]);
+                }
+                continue;
+            }
+
+            if (auto it = old_path->ops.find(item.first); it != old_path->ops.end()) {
+                if (item.second < it->second.size()) {
+                    Data& old_op = it->second[item.second];
+                    switch (item.first) {
+                        case Op::OpenChild:
+                            break;
+                        case Op::CloseChild:
+                            break;
+
+                        case Op::MatrixPush:
+                            Matrix_Push(&gGfxMatrix);
+                            break;
+
+                        case Op::MatrixPop:
+                            Matrix_Pop(&gGfxMatrix);
+                            break;
+
+                     // Unused on SF64
+                     // case Op::MatrixPut:
+                     //     interpolate_mtxf(&tmp_mtxf, &old_op.matrix_put.src, &new_op.matrix_put.src);
+                     //     Matrix_Put(&tmp_mtxf);
+                     //     break;
+
+                        case Op::MatrixMult:
+                            interpolate_mtxf(&tmp_mtxf, &old_op.matrix_mult.mf, &new_op.matrix_mult.mf);
+                            Matrix_Mult(gGfxMatrix, (Matrix*) &tmp_mtxf, new_op.matrix_mult.mode);
+                            break;
+
+                        case Op::MatrixTranslate:
+                            Matrix_Translate(gGfxMatrix, lerp(old_op.matrix_translate.x, new_op.matrix_translate.x),
+                                             lerp(old_op.matrix_translate.y, new_op.matrix_translate.y),
+                                             lerp(old_op.matrix_translate.z, new_op.matrix_translate.z),
+                                             new_op.matrix_translate.mode);
+                            break;
+
+                        case Op::MatrixScale:
+                            Matrix_Scale(gGfxMatrix, lerp(old_op.matrix_scale.x, new_op.matrix_scale.x),
+                                         lerp(old_op.matrix_scale.y, new_op.matrix_scale.y),
+                                         lerp(old_op.matrix_scale.z, new_op.matrix_scale.z), new_op.matrix_scale.mode);
+                            break;
+
+                        case Op::MatrixRotate1Coord: {
+                            float v = interpolate_angle(old_op.matrix_rotate_1_coord.value,
+                                                        new_op.matrix_rotate_1_coord.value);
+                            u8 mode = new_op.matrix_rotate_1_coord.mode;
+                            switch (new_op.matrix_rotate_1_coord.coord) {
+                                case 0:
+                                    Matrix_RotateX(gGfxMatrix, v, mode);
+                                    break;
+
+                                case 1:
+                                    Matrix_RotateY(gGfxMatrix, v, mode);
+                                    break;
+
+                                case 2:
+                                    Matrix_RotateZ(gGfxMatrix, v, mode);
+                                    break;
+                            }
+                            break;
+                        }
+                        case Op::MatrixMultVec3fNoTranslate: {
+                            interpolate_vecs(&tmp_vec3f, &old_op.matrix_vec_no_translate.src, &new_op.matrix_vec_no_translate.src);
+                            interpolate_vecs(&tmp_vec3f2, &old_op.matrix_vec_no_translate.dest, &new_op.matrix_vec_no_translate.dest);
+                            Matrix_MultVec3fNoTranslate(gCalcMatrix, &tmp_vec3f, &tmp_vec3f2);
+                            break;
+                        }
+                        case Op::MatrixMultVec3f: {
+                            interpolate_vecs(&tmp_vec3f, &old_op.matrix_vec_no_translate.src, &new_op.matrix_vec_no_translate.src);
+                            interpolate_vecs(&tmp_vec3f2, &old_op.matrix_vec_no_translate.dest, &new_op.matrix_vec_no_translate.dest);
+                            Matrix_MultVec3f(gCalcMatrix, &tmp_vec3f, &tmp_vec3f2);
+                            break;
+                        }
+
+                        case Op::MatrixMtxFToMtx:
+                            interpolate_mtxf(new_replacement(new_op.matrix_mtxf_to_mtx.dest),
+                                             &old_op.matrix_mtxf_to_mtx.src, &new_op.matrix_mtxf_to_mtx.src);
+                            break;
+
+                        case Op::MatrixToMtx: {
+                            //*new_replacement(new_op.matrix_to_mtx.dest) = *Matrix_GetCurrent();
+                            if (old_op.matrix_to_mtx.has_adjusted && new_op.matrix_to_mtx.has_adjusted) {
+                                interpolate_mtxf(&tmp_mtxf, &old_op.matrix_to_mtx.src, &new_op.matrix_to_mtx.src);
+                                Matrix_MtxFMtxFMult(&actor_mtx, &tmp_mtxf,
+                                                        new_replacement(new_op.matrix_to_mtx.dest));
+                            } else {
+                                interpolate_mtxf(new_replacement(new_op.matrix_to_mtx.dest), &old_op.matrix_to_mtx.src,
+                                                 &new_op.matrix_to_mtx.src);
+                            }
+                            break;
+                        }
+
+                        case Op::MatrixRotateAxis: {
+                            lerp_vec3f(&tmp_vec3f, &old_op.matrix_rotate_axis.axis, &new_op.matrix_rotate_axis.axis);
+                            auto tmp = interpolate_angle(old_op.matrix_rotate_axis.angle, new_op.matrix_rotate_axis.angle);
+                            Matrix_RotateAxis((Matrix*) &tmp_vec3f, tmp, 1.0f, 1.0f, 1.0f, new_op.matrix_rotate_axis.mode);
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    }
+};
+
+} // anonymous namespace
+
+unordered_map<Mtx*, MtxF> FrameInterpolation_Interpolate(float step) {
+    InterpolateCtx ctx;
+    ctx.step = step;
+    ctx.w = 1.0f - step;
+    ctx.interpolate_branch(&previous_recording.root_path, &current_recording.root_path);
+    return ctx.mtx_replacements;
+}
+
+bool camera_interpolation = true;
+
+void FrameInterpolation_ShouldInterpolateFrame(bool shouldInterpolate) {
+    camera_interpolation = shouldInterpolate;
+}
+
+void FrameInterpolation_StartRecord(void) {
+    previous_recording = move(current_recording);
+    current_recording = {};
+    current_path.clear();
+    current_path.push_back(&current_recording.root_path);
+    if (!camera_interpolation) {
+        // default to interpolating
+        camera_interpolation = true;
+        is_recording = false;
+        return;
+    }
+    if (GameEngine::GetInterpolationFPS() != 20) {
+        is_recording = true;
+    }
+}
+
+void FrameInterpolation_StopRecord(void) {
+    previous_camera_epoch = camera_epoch;
+    is_recording = false;
+}
+
+void FrameInterpolation_RecordOpenChild(const void* a, int b) {
+    if (!is_recording)
+        return;
+    label key = { a, b };
+    auto& m = current_path.back()->children[key];
+    append(Op::OpenChild).open_child = { key, m.size() };
+    current_path.push_back(&m.emplace_back());
+}
+
+void FrameInterpolation_RecordCloseChild(void) {
+    if (!is_recording)
+        return;
+    // append(Op::CloseChild);
+    if (has_inv_actor_mtx && current_path.size() == inv_actor_mtx_path_index) {
+        has_inv_actor_mtx = false;
+    }
+    current_path.pop_back();
+}
+
+void FrameInterpolation_DontInterpolateCamera(void) {
+    camera_epoch = previous_camera_epoch + 1;
+}
+
+int FrameInterpolation_GetCameraEpoch(void) {
+    return (int)camera_epoch;
+}
+
+void FrameInterpolation_RecordActorPosRotMatrix(void) {
+    if (!is_recording)
+        return;
+    next_is_actor_pos_rot_matrix = true;
+}
+
+void FrameInterpolation_RecordMatrixPush(void) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixPush);
+}
+
+void FrameInterpolation_RecordMatrixPop(void) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixPop);
+}
+
+void FrameInterpolation_RecordMatrixPut(MtxF* src) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixPut).matrix_put = { *src };
+}
+
+void FrameInterpolation_RecordMatrixMult(MtxF* mf, u8 mode) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixMult).matrix_mult = { *mf, mode };
+}
+
+void FrameInterpolation_RecordMatrixTranslate(f32 x, f32 y, f32 z, u8 mode) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixTranslate).matrix_translate = { x, y, z, mode };
+}
+
+void FrameInterpolation_RecordMatrixScale(f32 x, f32 y, f32 z, u8 mode) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixScale).matrix_scale = { x, y, z, mode };
+}
+
+void FrameInterpolation_RecordMatrixMultVec3fNoTranslate(Vec3f src, Vec3f dest){
+    if (!is_recording)
+        return;
+    append(Op::MatrixMultVec3fNoTranslate).matrix_vec_no_translate = { src, dest };
+}
+
+void FrameInterpolation_RecordMatrixMultVec3f(Vec3f src, Vec3f dest){
+    if (!is_recording)
+        return;
+    append(Op::MatrixMultVec3f).matrix_vec_translate = { src, dest };
+}
+
+void FrameInterpolation_RecordMatrixRotate1Coord(u32 coord, f32 value, u8 mode) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixRotate1Coord).matrix_rotate_1_coord = { coord, value, mode };
+}
+
+void FrameInterpolation_RecordMatrixMtxFToMtx(MtxF* src, Mtx* dest) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixMtxFToMtx).matrix_mtxf_to_mtx = { *src, dest };
+}
+
+void FrameInterpolation_RecordMatrixToMtx(Mtx* dest, char* file, s32 line) {
+    if (!is_recording)
+        return;
+    auto& d = append(Op::MatrixToMtx).matrix_to_mtx = { dest };
+    if (has_inv_actor_mtx) {
+        d.has_adjusted = true;
+        Matrix_MtxFMtxFMult(&inv_actor_mtx, Matrix_GetCurrent(), &d.src);
+    } else {
+        d.src = *Matrix_GetCurrent();
+    }
+}
+
+void FrameInterpolation_RecordMatrixRotateAxis(f32 angle, Vec3f* axis, u8 mode) {
+    if (!is_recording)
+        return;
+    append(Op::MatrixRotateAxis).matrix_rotate_axis = { angle, *axis, mode };
+}
+
+void FrameInterpolation_RecordSkinMatrixMtxFToMtx(MtxF* src, Mtx* dest) {
+    if (!is_recording)
+        return;
+    FrameInterpolation_RecordMatrixMtxFToMtx(src, dest);
+}
+
+// https://stackoverflow.com/questions/1148309/inverting-a-4x4-matrix
+static bool invert_matrix(const float m[16], float invOut[16]) {
+    float inv[16], det;
+    int i;
+
+    inv[0] = m[5] * m[10] * m[15] - m[5] * m[11] * m[14] - m[9] * m[6] * m[15] + m[9] * m[7] * m[14] +
+             m[13] * m[6] * m[11] - m[13] * m[7] * m[10];
+
+    inv[4] = -m[4] * m[10] * m[15] + m[4] * m[11] * m[14] + m[8] * m[6] * m[15] - m[8] * m[7] * m[14] -
+             m[12] * m[6] * m[11] + m[12] * m[7] * m[10];
+
+    inv[8] = m[4] * m[9] * m[15] - m[4] * m[11] * m[13] - m[8] * m[5] * m[15] + m[8] * m[7] * m[13] +
+             m[12] * m[5] * m[11] - m[12] * m[7] * m[9];
+
+    inv[12] = -m[4] * m[9] * m[14] + m[4] * m[10] * m[13] + m[8] * m[5] * m[14] - m[8] * m[6] * m[13] -
+              m[12] * m[5] * m[10] + m[12] * m[6] * m[9];
+
+    inv[1] = -m[1] * m[10] * m[15] + m[1] * m[11] * m[14] + m[9] * m[2] * m[15] - m[9] * m[3] * m[14] -
+             m[13] * m[2] * m[11] + m[13] * m[3] * m[10];
+
+    inv[5] = m[0] * m[10] * m[15] - m[0] * m[11] * m[14] - m[8] * m[2] * m[15] + m[8] * m[3] * m[14] +
+             m[12] * m[2] * m[11] - m[12] * m[3] * m[10];
+
+    inv[9] = -m[0] * m[9] * m[15] + m[0] * m[11] * m[13] + m[8] * m[1] * m[15] - m[8] * m[3] * m[13] -
+             m[12] * m[1] * m[11] + m[12] * m[3] * m[9];
+
+    inv[13] = m[0] * m[9] * m[14] - m[0] * m[10] * m[13] - m[8] * m[1] * m[14] + m[8] * m[2] * m[13] +
+              m[12] * m[1] * m[10] - m[12] * m[2] * m[9];
+
+    inv[2] = m[1] * m[6] * m[15] - m[1] * m[7] * m[14] - m[5] * m[2] * m[15] + m[5] * m[3] * m[14] +
+             m[13] * m[2] * m[7] - m[13] * m[3] * m[6];
+
+    inv[6] = -m[0] * m[6] * m[15] + m[0] * m[7] * m[14] + m[4] * m[2] * m[15] - m[4] * m[3] * m[14] -
+             m[12] * m[2] * m[7] + m[12] * m[3] * m[6];
+
+    inv[10] = m[0] * m[5] * m[15] - m[0] * m[7] * m[13] - m[4] * m[1] * m[15] + m[4] * m[3] * m[13] +
+              m[12] * m[1] * m[7] - m[12] * m[3] * m[5];
+
+    inv[14] = -m[0] * m[5] * m[14] + m[0] * m[6] * m[13] + m[4] * m[1] * m[14] - m[4] * m[2] * m[13] -
+              m[12] * m[1] * m[6] + m[12] * m[2] * m[5];
+
+    inv[3] = -m[1] * m[6] * m[11] + m[1] * m[7] * m[10] + m[5] * m[2] * m[11] - m[5] * m[3] * m[10] -
+             m[9] * m[2] * m[7] + m[9] * m[3] * m[6];
+
+    inv[7] = m[0] * m[6] * m[11] - m[0] * m[7] * m[10] - m[4] * m[2] * m[11] + m[4] * m[3] * m[10] +
+             m[8] * m[2] * m[7] - m[8] * m[3] * m[6];
+
+    inv[11] = -m[0] * m[5] * m[11] + m[0] * m[7] * m[9] + m[4] * m[1] * m[11] - m[4] * m[3] * m[9] -
+              m[8] * m[1] * m[7] + m[8] * m[3] * m[5];
+
+    inv[15] = m[0] * m[5] * m[10] - m[0] * m[6] * m[9] - m[4] * m[1] * m[10] + m[4] * m[2] * m[9] + m[8] * m[1] * m[6] -
+              m[8] * m[2] * m[5];
+
+    det = m[0] * inv[0] + m[1] * inv[4] + m[2] * inv[8] + m[3] * inv[12];
+
+    if (det == 0) {
+        return false;
+    }
+
+    det = 1.0 / det;
+
+    for (i = 0; i < 16; i++) {
+        invOut[i] = inv[i] * det;
+    }
+
+    return true;
+}

src/port/interpolation/FrameInterpolation.h

@@ -0,0 +1,59 @@
+#pragma once
+
+#include "sf64math.h"
+#include <libultraship.h>
+
+#ifdef __cplusplus
+
+#include <unordered_map>
+
+std::unordered_map<Mtx*, MtxF> FrameInterpolation_Interpolate(float step);
+
+extern "C" {
+
+#endif
+
+void FrameInterpolation_ShouldInterpolateFrame(bool shouldInterpolate);
+
+void FrameInterpolation_StartRecord(void);
+
+void FrameInterpolation_StopRecord(void);
+
+void FrameInterpolation_RecordOpenChild(const void* a, int b);
+
+void FrameInterpolation_RecordCloseChild(void);
+
+void FrameInterpolation_DontInterpolateCamera(void);
+
+int FrameInterpolation_GetCameraEpoch(void);
+
+void FrameInterpolation_RecordActorPosRotMatrix(void);
+
+void FrameInterpolation_RecordMatrixPush(void);
+
+void FrameInterpolation_RecordMatrixPop(void);
+
+void FrameInterpolation_RecordMatrixMult(MtxF* mf, u8 mode);
+
+void FrameInterpolation_RecordMatrixTranslate(f32 x, f32 y, f32 z, u8 mode);
+
+void FrameInterpolation_RecordMatrixScale(f32 x, f32 y, f32 z, u8 mode);
+
+void FrameInterpolation_RecordMatrixRotate1Coord(u32 coord, f32 value, u8 mode);
+
+void FrameInterpolation_RecordMatrixMtxFToMtx(MtxF* src, Mtx* dest);
+
+void FrameInterpolation_RecordMatrixToMtx(Mtx* dest, char* file, s32 line);
+
+void FrameInterpolation_RecordMatrixReplaceRotation(MtxF* mf);
+
+void FrameInterpolation_RecordMatrixRotateAxis(f32 angle, Vec3f* axis, u8 mode);
+
+void FrameInterpolation_RecordSkinMatrixMtxFToMtx(MtxF* src, Mtx* dest);
+
+void FrameInterpolation_RecordMatrixMultVec3f(Vec3f src, Vec3f dest);
+void FrameInterpolation_RecordMatrixMultVec3fNoTranslate(Vec3f src, Vec3f dest);
+
+#ifdef __cplusplus
+}
+#endif

src/port/ui/ImguiUI.cpp

@@ -252,7 +252,7 @@ void DrawSettingsMenu(){
                 currentFps = 60;
             }
             CVarSetInteger("gInterpolationFPS", currentFps);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
         #else
             bool matchingRefreshRate =
                 CVarGetInteger("gMatchRefreshRate", 0) && Ship::Context::GetInstance()->GetWindow()->GetWindowBackend() != Ship::WindowBackend::FAST3D_DXGI_DX11;
@@ -280,7 +280,7 @@ void DrawSettingsMenu(){
                 int hz = Ship::Context::GetInstance()->GetWindow()->GetCurrentRefreshRate();
                 if (hz >= 30 && hz <= 360) {
                     CVarSetInteger("gInterpolationFPS", hz);
-                    Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+                    Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
                 }
             }
         } else {

src/port/ui/UIWidgets.cpp

@@ -205,7 +205,7 @@ namespace UIWidgets {
         bool val = (bool)CVarGetInteger(cvarName, defaultValue);
         if (CustomCheckbox(text, &val, disabled, disabledGraphic)) {
             CVarSetInteger(cvarName, val);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             changed = true;
         }
 
@@ -245,7 +245,7 @@ namespace UIWidgets {
                         CVarSetInteger(cvarName, i);
                         selected = i;
                         changed = true;
-                        Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+                        Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
                     }
                 }
             }
@@ -258,7 +258,7 @@ namespace UIWidgets {
             if (disabledValue >= 0 && selected != disabledValue) {
                 CVarSetInteger(cvarName, disabledValue);
                 changed = true;
-                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             }
         }
 
@@ -347,7 +347,7 @@ namespace UIWidgets {
 
         if (changed) {
             CVarSetInteger(cvarName, val);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
         }
 
         return changed;
@@ -423,7 +423,7 @@ namespace UIWidgets {
 
         if (changed) {
             CVarSetFloat(cvarName, val);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
         }
 
         return changed;
@@ -470,7 +470,7 @@ namespace UIWidgets {
         int val = CVarGetInteger(cvarName, 0);
         if (ImGui::RadioButton(make_invisible.c_str(), id == val)) {
             CVarSetInteger(cvarName, id);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             ret = true;
         }
         ImGui::SameLine();
@@ -497,7 +497,7 @@ namespace UIWidgets {
 
             CVarSetColor(cvarName, colorsRGBA);
             CVarSetInteger(Cvar_RBM.c_str(), 0); //On click disable rainbow mode.
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             changed = true;
         }
         Tooltip("Revert colors to the game's original colors (GameCube version)\nOverwrites previously chosen color");
@@ -763,7 +763,7 @@ namespace UIWidgets {
         bool value = (bool)CVarGetInteger(cvarName, options.defaultValue);
         if (Checkbox(label, &value, options)) {
             CVarSetInteger(cvarName, value);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             dirty = true;
         }
         return dirty;
@@ -867,7 +867,7 @@ namespace UIWidgets {
         uint8_t value = (uint8_t)CVarGetInteger(cvarName, options.defaultIndex);
         if (Combobox(label, &value, comboArray, options)) {
             CVarSetInteger(cvarName, value);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             dirty = true;
         }
         return dirty;
@@ -914,7 +914,7 @@ namespace UIWidgets {
             if (Button("-", { .color = options.color, .size = Sizes::Inline }) && *value > min) {
                 *value -= options.step;
                 if (*value < min) *value = min;
-                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
                 dirty = true;
             }
             ImGui::SameLine(0, 3.0f);
@@ -923,7 +923,7 @@ namespace UIWidgets {
             ImGui::SetNextItemWidth(ImGui::GetContentRegionAvail().x);
         }
         if (ImGui::SliderScalar(invisibleLabel, ImGuiDataType_S32, value, &min, &max, options.format, options.flags)) {
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             dirty = true;
         }
         if (options.showButtons) {
@@ -932,7 +932,7 @@ namespace UIWidgets {
             if (Button("+", { .color = options.color, .size = Sizes::Inline }) && *value < max) {
                 *value += options.step;
                 if (*value > max) *value = max;
-                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
                 dirty = true;
             }
         }
@@ -953,7 +953,7 @@ namespace UIWidgets {
         int32_t value = CVarGetInteger(cvarName, defaultValue);
         if (SliderInt(label, &value, min, max, options)) {
             CVarSetInteger(cvarName, value);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             dirty = true;
         }
         return dirty;
@@ -985,7 +985,7 @@ namespace UIWidgets {
             if (Button("-", { .color = options.color, .size = Sizes::Inline }) && *value > min) {
                 *value -= options.step;
                 if (*value < min) *value = min;
-                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
                 dirty = true;
             }
             ImGui::SameLine(0, 3.0f);
@@ -995,7 +995,7 @@ namespace UIWidgets {
         }
         if (ImGui::SliderScalar(invisibleLabel, ImGuiDataType_Float, &valueToDisplay, &minToDisplay, &maxToDisplay, options.format, options.flags)) {
             *value = options.isPercentage ? valueToDisplay / 100.0f : valueToDisplay;
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             dirty = true;
         }
         if (options.showButtons) {
@@ -1004,7 +1004,7 @@ namespace UIWidgets {
             if (Button("+", { .color = options.color, .size = Sizes::Inline }) && *value < max) {
                 *value += options.step;
                 if (*value > max) *value = max;
-                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+                Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
                 dirty = true;
             }
         }
@@ -1025,7 +1025,7 @@ namespace UIWidgets {
         float value = CVarGetFloat(cvarName, defaultValue);
         if (SliderFloat(label, &value, min, max, options)) {
             CVarSetFloat(cvarName, value);
-            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesOnNextTick();
+            Ship::Context::GetInstance()->GetWindow()->GetGui()->SaveConsoleVariablesNextFrame();
             dirty = true;
         }
         return dirty;

src/sys/sys_lib.c

@@ -1,4 +1,5 @@
 #include "sys.h"
+#include "port/interpolation/FrameInterpolation.h"
 
 s32 Lib_vsPrintf(char* dst, const char* fmt, va_list args) {
     return vsprintf(dst, fmt, args);
@@ -84,21 +85,25 @@ void Lib_QuickSort(u8* first, u32 length, u32 size, CompareFunc cFunc) {
 void Lib_InitPerspective(Gfx** dList) {
     u16 norm;
 
+    FrameInterpolation_RecordOpenChild("perspective", 0);
     guPerspective(gGfxMtx, &norm, gFovY, (f32) SCREEN_WIDTH / SCREEN_HEIGHT, gProjectNear, gProjectFar, 1.0f);
     gSPPerspNormalize((*dList)++, norm);
     gSPMatrix((*dList)++, gGfxMtx++, G_MTX_NOPUSH | G_MTX_LOAD | G_MTX_PROJECTION);
     guLookAt(gGfxMtx, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, -12800.0f, 0.0f, 1.0f, 0.0f);
     gSPMatrix((*dList)++, gGfxMtx++, G_MTX_NOPUSH | G_MTX_MUL | G_MTX_PROJECTION);
     Matrix_Copy(gGfxMatrix, &gIdentityMatrix);
+    FrameInterpolation_RecordCloseChild();
 }
 
 void Lib_InitOrtho(Gfx** dList) {
+    FrameInterpolation_RecordOpenChild("ortho", 0);
     guOrtho(gGfxMtx, -SCREEN_WIDTH / 2, SCREEN_WIDTH / 2, -SCREEN_HEIGHT / 2, SCREEN_HEIGHT / 2, gProjectNear,
             gProjectFar, 1.0f);
     gSPMatrix((*dList)++, gGfxMtx++, G_MTX_NOPUSH | G_MTX_LOAD | G_MTX_PROJECTION);
     guLookAt(gGfxMtx, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, -12800.0f, 0.0f, 1.0f, 0.0f);
     gSPMatrix((*dList)++, gGfxMtx++, G_MTX_NOPUSH | G_MTX_MUL | G_MTX_PROJECTION);
     Matrix_Copy(gGfxMatrix, &gIdentityMatrix);
+    FrameInterpolation_RecordCloseChild();
 }
 
 void Lib_DmaRead(void* src, void* dst, s32 size) {

src/sys/sys_matrix.c

@@ -1,5 +1,6 @@
 #include "sys.h"
 
+#include "port/interpolation/FrameInterpolation.h"
 #define qs1616(e) ((s32) ((e) *0x00010000))
 
 #define IPART(x) ((qs1616(x) >> 16) & 0xFFFF)
@@ -46,17 +47,152 @@ void Matrix_Copy(Matrix* dst, Matrix* src) {
 
 // Makes a copy of the stack's current matrix and puts it on the top of the stack
 void Matrix_Push(Matrix** mtxStack) {
+    FrameInterpolation_RecordMatrixPush();
     Matrix_Copy(*mtxStack + 1, *mtxStack);
     (*mtxStack)++;
 }
 
 // Removes the top matrix of the stack
 void Matrix_Pop(Matrix** mtxStack) {
+    FrameInterpolation_RecordMatrixPop();
     (*mtxStack)--;
 }
 
+/**
+ * Matrix multiplication, dest = mfA * mfB.
+ * mfA and dest should not be the same matrix.
+ */
+void Matrix_MtxFMtxFMult(MtxF* mfB, MtxF* mfA, MtxF* dest) {
+    f32 rx;
+    f32 ry;
+    f32 rz;
+    f32 rw;
+
+    //---COL1---
+    f32 cx = mfB->xx;
+    f32 cy = mfB->xy;
+    f32 cz = mfB->xz;
+    f32 cw = mfB->xw;
+    //--------
+
+    rx = mfA->xx;
+    ry = mfA->yx;
+    rz = mfA->zx;
+    rw = mfA->wx;
+    dest->xx = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xy;
+    ry = mfA->yy;
+    rz = mfA->zy;
+    rw = mfA->wy;
+    dest->xy = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xz;
+    ry = mfA->yz;
+    rz = mfA->zz;
+    rw = mfA->wz;
+    dest->xz = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xw;
+    ry = mfA->yw;
+    rz = mfA->zw;
+    rw = mfA->ww;
+    dest->xw = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    //---2Col---
+    cx = mfB->yx;
+    cy = mfB->yy;
+    cz = mfB->yz;
+    cw = mfB->yw;
+    //--------
+    rx = mfA->xx;
+    ry = mfA->yx;
+    rz = mfA->zx;
+    rw = mfA->wx;
+    dest->yx = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xy;
+    ry = mfA->yy;
+    rz = mfA->zy;
+    rw = mfA->wy;
+    dest->yy = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xz;
+    ry = mfA->yz;
+    rz = mfA->zz;
+    rw = mfA->wz;
+    dest->yz = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xw;
+    ry = mfA->yw;
+    rz = mfA->zw;
+    rw = mfA->ww;
+    dest->yw = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    //---3Col---
+    cx = mfB->zx;
+    cy = mfB->zy;
+    cz = mfB->zz;
+    cw = mfB->zw;
+    //--------
+    rx = mfA->xx;
+    ry = mfA->yx;
+    rz = mfA->zx;
+    rw = mfA->wx;
+    dest->zx = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xy;
+    ry = mfA->yy;
+    rz = mfA->zy;
+    rw = mfA->wy;
+    dest->zy = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xz;
+    ry = mfA->yz;
+    rz = mfA->zz;
+    rw = mfA->wz;
+    dest->zz = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xw;
+    ry = mfA->yw;
+    rz = mfA->zw;
+    rw = mfA->ww;
+    dest->zw = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    //---4Col---
+    cx = mfB->wx;
+    cy = mfB->wy;
+    cz = mfB->wz;
+    cw = mfB->ww;
+    //--------
+    rx = mfA->xx;
+    ry = mfA->yx;
+    rz = mfA->zx;
+    rw = mfA->wx;
+    dest->wx = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xy;
+    ry = mfA->yy;
+    rz = mfA->zy;
+    rw = mfA->wy;
+    dest->wy = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xz;
+    ry = mfA->yz;
+    rz = mfA->zz;
+    rw = mfA->wz;
+    dest->wz = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+
+    rx = mfA->xw;
+    ry = mfA->yw;
+    rz = mfA->zw;
+    rw = mfA->ww;
+    dest->ww = (cx * rx) + (cy * ry) + (cz * rz) + (cw * rw);
+}
+
 // Copies tf into mtx (MTXF_NEW) or applies it to mtx (MTXF_APPLY)
 void Matrix_Mult(Matrix* mtx, Matrix* tf, u8 mode) {
+    FrameInterpolation_RecordMatrixMult(tf, mode);
     f32 rx;
     f32 ry;
     f32 rz;
@@ -109,6 +245,7 @@ void Matrix_Mult(Matrix* mtx, Matrix* tf, u8 mode) {
 
 // Creates a translation matrix in mtx (MTXF_NEW) or applies one to mtx (MTXF_APPLY)
 void Matrix_Translate(Matrix* mtx, f32 x, f32 y, f32 z, u8 mode) {
+    FrameInterpolation_RecordMatrixTranslate(x, y, z, mode);
     f32 rx;
     f32 ry;
     s32 i;
@@ -132,6 +269,7 @@ void Matrix_Translate(Matrix* mtx, f32 x, f32 y, f32 z, u8 mode) {
 
 // Creates a scale matrix in mtx (MTXF_NEW) or applies one to mtx (MTXF_APPLY)
 void Matrix_Scale(Matrix* mtx, f32 xScale, f32 yScale, f32 zScale, u8 mode) {
+    FrameInterpolation_RecordMatrixScale(xScale, yScale, zScale, mode);
     f32 rx;
     f32 ry;
     s32 i;
@@ -157,6 +295,7 @@ void Matrix_Scale(Matrix* mtx, f32 xScale, f32 yScale, f32 zScale, u8 mode) {
 
 // Creates rotation matrix about the X axis in mtx (MTXF_NEW) or applies one to mtx (MTXF_APPLY)
 void Matrix_RotateX(Matrix* mtx, f32 angle, u8 mode) {
+    FrameInterpolation_RecordMatrixRotate1Coord(0, angle, mode);
     f32 cs;
     f32 sn;
     f32 ry;
@@ -185,6 +324,7 @@ void Matrix_RotateX(Matrix* mtx, f32 angle, u8 mode) {
 
 // Creates rotation matrix about the Y axis in mtx (MTXF_NEW) or applies one to mtx (MTXF_APPLY)
 void Matrix_RotateY(Matrix* mtx, f32 angle, u8 mode) {
+    FrameInterpolation_RecordMatrixRotate1Coord(1, angle, mode);
     f32 cs;
     f32 sn;
     f32 rx;
@@ -213,6 +353,7 @@ void Matrix_RotateY(Matrix* mtx, f32 angle, u8 mode) {
 
 // Creates rotation matrix about the Z axis in mtx (MTXF_NEW) or applies one to mtx (MTXF_APPLY)
 void Matrix_RotateZ(Matrix* mtx, f32 angle, u8 mode) {
+    FrameInterpolation_RecordMatrixRotate1Coord(2, angle, mode);
     f32 cs;
     f32 sn;
     f32 rx;
@@ -242,6 +383,7 @@ void Matrix_RotateZ(Matrix* mtx, f32 angle, u8 mode) {
 // Creates rotation matrix about a given vector axis in mtx (MTXF_NEW) or applies one to mtx (MTXF_APPLY).
 // The vector specifying the axis does not need to be a unit vector.
 void Matrix_RotateAxis(Matrix* mtx, f32 angle, f32 axisX, f32 axisY, f32 axisZ, u8 mode) {
+//    FrameInterpolation_RecordMatrixRotateAxis()
     f32 rx;
     f32 ry;
     f32 rz;
@@ -336,17 +478,20 @@ void Matrix_RotateAxis(Matrix* mtx, f32 angle, f32 axisX, f32 axisY, f32 axisZ,
 
 // Converts the current Gfx matrix to a Mtx
 void Matrix_ToMtx(Mtx* dest) {
+    FrameInterpolation_RecordMatrixMtxFToMtx(gGfxMatrix->m, dest);
     // LTODO: We need to validate this
     guMtxF2L(gGfxMatrix->m, dest);
 }
 
 // Converts the Mtx src to a Matrix, putting the result in dest
 void Matrix_FromMtx(Mtx* src, Matrix* dest) {
+    FrameInterpolation_RecordMatrixMtxFToMtx(src, dest);
     guMtxF2L(src->m, dest->m);
 }
 
 // Applies the transform matrix mtx to the vector src, putting the result in dest
 void Matrix_MultVec3f(Matrix* mtx, Vec3f* src, Vec3f* dest) {
+    FrameInterpolation_RecordMatrixMultVec3f(*src, *dest);
     dest->x = (mtx->m[0][0] * src->x) + (mtx->m[1][0] * src->y) + (mtx->m[2][0] * src->z) + mtx->m[3][0];
     dest->y = (mtx->m[0][1] * src->x) + (mtx->m[1][1] * src->y) + (mtx->m[2][1] * src->z) + mtx->m[3][1];
     dest->z = (mtx->m[0][2] * src->x) + (mtx->m[1][2] * src->y) + (mtx->m[2][2] * src->z) + mtx->m[3][2];
@@ -355,6 +500,7 @@ void Matrix_MultVec3f(Matrix* mtx, Vec3f* src, Vec3f* dest) {
 // Applies the linear part of the transformation matrix mtx to the vector src, ignoring any translation that mtx might
 // have. Puts the result in dest.
 void Matrix_MultVec3fNoTranslate(Matrix* mtx, Vec3f* src, Vec3f* dest) {
+    FrameInterpolation_RecordMatrixMultVec3fNoTranslate(*src, *dest);
     dest->x = (mtx->m[0][0] * src->x) + (mtx->m[1][0] * src->y) + (mtx->m[2][0] * src->z);
     dest->y = (mtx->m[0][1] * src->x) + (mtx->m[1][1] * src->y) + (mtx->m[2][1] * src->z);
     dest->z = (mtx->m[0][2] * src->x) + (mtx->m[1][2] * src->y) + (mtx->m[2][2] * src->z);