libradhook/tests/test_vtable.cpp

#include <cassert>
#include "radhook/radhook.h"

struct IFoo {
    virtual int Value() { return 7; }
    virtual ~IFoo() = default;
};

static int DetourValue(IFoo* /*self*/) { return 777; }

using Fn = int(*)(IFoo*);

int main() {
    IFoo obj;
    void** vtable = *reinterpret_cast<void***>(&obj);

    RadHookHandle h = nullptr;
    assert(RadHookVTableCreate(vtable, 0, (void*)DetourValue, &h) == RadHookResult::Success);
    assert(RadHookIsValid(h));
    assert(RadHookIsEnabled(h));

    // handle is created enabled, so the swap must already be visible
    assert(obj.Value() == 777);
    assert(reinterpret_cast<Fn>(vtable[0])(&obj) == 777);

    void* originalEntry = RadHookGetTarget(h);
    assert(originalEntry != nullptr);
    assert(RadHookGetDetour(h) == reinterpret_cast<void*>(DetourValue));

    // disable restores the original vtable entry
    assert(RadHookDisable(h) == RadHookResult::Success);
    assert(!RadHookIsEnabled(h));
    assert(obj.Value() == 7);
    assert(vtable[0] == originalEntry);

    // re-enable re-applies the swap
    assert(RadHookEnable(h) == RadHookResult::Success);
    assert(obj.Value() == 777);

    // no relocation is needed for vtable hooks -- the trampoline is the
    // original entry itself and can be called directly while the hook is live
    auto original = RadHookGetOriginalAs<Fn>(h);
    assert(reinterpret_cast<void*>(original) == originalEntry);
    assert(original(&obj) == 7);

    RadHookDestroy(h);
    assert(!RadHookIsValid(h));
    assert(obj.Value() == 7);

    // queueing works uniformly across inline and vtable hook kinds
    RadHookHandle h2 = nullptr;
    assert(RadHookVTableCreate(vtable, 0, (void*)DetourValue, &h2) == RadHookResult::Success);
    assert(RadHookQueueDisable(h2) == RadHookResult::Success);
    assert(RadHookIsEnabled(h2)); // not applied yet
    assert(RadHookApplyQueued() == RadHookResult::Success);
    assert(!RadHookIsEnabled(h2));
    assert(obj.Value() == 7);
    RadHookDestroy(h2);

    // invalid inputs
    RadHookHandle bad = nullptr;
    assert(RadHookVTableCreate(nullptr, 0, (void*)DetourValue, &bad) == RadHookResult::InvalidTarget);
    assert(bad == nullptr);
    assert(RadHookVTableCreate(vtable, 0, nullptr, &bad) == RadHookResult::InvalidDetour);
    assert(bad == nullptr);

    return 0;
}