api changes, check README

tests/test_vtable.cpp

@@ -0,0 +1,68 @@
+#include <cassert>
+#include "radhook/radhook.h"
+
+struct IFoo {
+    virtual int Value() { return 7; }
+    virtual ~IFoo() = default;
+};
+
+static int DetourValue(IFoo* /*self*/) { return 777; }
+
+using Fn = int(*)(IFoo*);
+
+int main() {
+    IFoo obj;
+    void** vtable = *reinterpret_cast<void***>(&obj);
+
+    RadHookHandle h = nullptr;
+    assert(RadHookVTableCreate(vtable, 0, (void*)DetourValue, &h) == RadHookResult::Success);
+    assert(RadHookIsValid(h));
+    assert(RadHookIsEnabled(h));
+
+    // handle is created enabled, so the swap must already be visible
+    assert(obj.Value() == 777);
+    assert(reinterpret_cast<Fn>(vtable[0])(&obj) == 777);
+
+    void* originalEntry = RadHookGetTarget(h);
+    assert(originalEntry != nullptr);
+    assert(RadHookGetDetour(h) == reinterpret_cast<void*>(DetourValue));
+
+    // disable restores the original vtable entry
+    assert(RadHookDisable(h) == RadHookResult::Success);
+    assert(!RadHookIsEnabled(h));
+    assert(obj.Value() == 7);
+    assert(vtable[0] == originalEntry);
+
+    // re-enable re-applies the swap
+    assert(RadHookEnable(h) == RadHookResult::Success);
+    assert(obj.Value() == 777);
+
+    // no relocation is needed for vtable hooks -- the trampoline is the
+    // original entry itself and can be called directly while the hook is live
+    auto original = RadHookGetOriginalAs<Fn>(h);
+    assert(reinterpret_cast<void*>(original) == originalEntry);
+    assert(original(&obj) == 7);
+
+    RadHookDestroy(h);
+    assert(!RadHookIsValid(h));
+    assert(obj.Value() == 7);
+
+    // queueing works uniformly across inline and vtable hook kinds
+    RadHookHandle h2 = nullptr;
+    assert(RadHookVTableCreate(vtable, 0, (void*)DetourValue, &h2) == RadHookResult::Success);
+    assert(RadHookQueueDisable(h2) == RadHookResult::Success);
+    assert(RadHookIsEnabled(h2)); // not applied yet
+    assert(RadHookApplyQueued() == RadHookResult::Success);
+    assert(!RadHookIsEnabled(h2));
+    assert(obj.Value() == 7);
+    RadHookDestroy(h2);
+
+    // invalid inputs
+    RadHookHandle bad = nullptr;
+    assert(RadHookVTableCreate(nullptr, 0, (void*)DetourValue, &bad) == RadHookResult::InvalidTarget);
+    assert(bad == nullptr);
+    assert(RadHookVTableCreate(vtable, 0, nullptr, &bad) == RadHookResult::InvalidDetour);
+    assert(bad == nullptr);
+
+    return 0;
+}