Dominik Grzelak blog

Source code for projecting a ray from screen coordinates for mouse picking in conjunction with Irrlicht and Bullet Physics for collision checks.
Mathematics from: https://www.mvps.org/directx/articles/rayproj.htm

Snippet:
matrix4 unprojection = camera->getViewMatrix();
unprojection.makeInverse();

float dx = tanf(camera->getFOV() * 0.5f) * (MouseState.Position.X / (screenWidth * 0.5f) - 1.0f) / camera->getAspectRatio();
float dy = tanf(camera->getFOV() * 0.5f) * (1.0f - MouseState.Position.Y / (screenHeight * 0.5f));
vector3df p1(dx * camera->getNearValue(), dy*camera->getNearValue(), camera->getNearValue());
vector3df p2(dx * camera->getFarValue(), dy*camera->getFarValue(), camera->getFarValue());

unprojection.transformVect(p1);
unprojection.transformVect(p2);

vector3df dir = p2 - p1;
dir = dir.normalize();

vector3df rayWorldStart = p1;
vector3df rayWorldEnd = p1 + dir * 20;

btCollisionWorld::ClosestRayResultCallback RayCallback(BulletTransformHelper::convertVector3(rayWorldStart), BulletTransformHelper::convertVector3(rayWorldEnd));
AState::getPhysicHandler()->getWorld()->rayTest(BulletTransformHelper::convertVector3(rayWorldStart),
BulletTransformHelper::convertVector3(rayWorldEnd),
RayCallback);
selectedNode = 0;
if (RayCallback.hasHit()) {
btRigidBody* rigidBody = (btRigidBody*)(RayCallback.m_collisionObject);
selectedNode = rigidBody->getUserPointer();
}