Stereoscopic test

Yesterday I did a small test with stereo rendering, trying to reproject the image from one eye to the other. It's very crude but I wanted to show the "theory" or the basic idea of the procedure as I got a few people asking me about it.

Stereo rendering (standard red/cyan glasses needed)

Of course the most interesting thing is how to correctly inpaint the holes in your visibility. This is a very general problem that also applies to other post effects like motion blur and depth of field blur, the solution I use in this test well... I won't even call it a solution. 

Much better methods have to be used for this to work decently, but that will (might...) be the topic of another post!

Reconstructed view

Rendered view

 float Script : STANDARDSGLOBAL <
    string UIWidget = "none";
    string ScriptClass = "scene";
    string ScriptOrder = "standard";
    string ScriptOutput = "color";
    string Script = "Technique=Main;";
> = 0.8; // version #

// -- Untweakables

float4x4 WorldViewProj : WorldViewProjection < string UIWidget="None"; >;
float4x4 WorldView : WorldView < string UIWidget="None"; >;
float4x4 Proj : Projection ;

float2 BufferSize : ViewportPixelSize < string UIWidget="None"; >;

float4 ClearParam < string UIWidget="None"; > = {0,0,0,0};

// -- Buffers and samplers

texture Bake : RENDERCOLORTARGET
<
    float2 ViewPortRatio = {1,1};
    string Format = "A8B8G8R8";
    string UIWidget = "None";
>;
sampler2D BakeSampler = sampler_state
{
    texture = ;
    MagFilter = Point;
    MinFilter = Point;
    AddressU = Clamp;
    AddressV = Clamp;
    MipFilter = Point;
};

// -- Data structures

struct GeomVS_Out
{
    float4 Pos : POSITION;
    float4 PosCopy : TEXCOORD0;
    float2 UV : TEXCOORD2;
};

struct FSQuadVS_InOut // fullscreen quad
{
    float4 Pos : POSITION;
    float2 UV : TEXCOORD0;
};

// -- Shaders

GeomVS_Out GeomVS(float4 pos : POSITION, float2 uv : TEXCOORD0)
{
    GeomVS_Out Out;

    Out.Pos = mul( pos, WorldViewProj );
    Out.PosCopy = Out.Pos;
    Out.UV = uv;
   
    return Out;
}

FSQuadVS_InOut FSQuadVS(FSQuadVS_InOut In)
{   
    return In;
}

// FX Composer does not pass near and far planes via a semantic, we have to "extract" them
float2 GetPlanesFromDxProjection()
{
    float near = -Proj[3][2] / Proj[2][2];
    float far = -Proj[3][2] / (Proj[2][2] - 1.0);
   
    return float2(near, far);
}

float4 EncUnitRangeToRGBA8( float v )
{
    float4 enc = float4(1.0, 255.0, 65025.0, 160581375.0) * v;
   
    enc = frac(enc);
    enc -= enc.yzww * float4(1.0/255.0, 1.0/255.0, 1.0/255.0, 0.0);
   
    return enc;
}

float DecUnitRangeToRGBA8( float4 rgba )
{
    return dot( rgba, float4(1.0, 1/255.0, 1/65025.0, 1/160581375.0) );
}

float4 BakePS(GeomVS_Out In) : COLOR
{
    // 0 to 1 from near to far
    float linearDepth = In.PosCopy.z / GetPlanesFromDxProjection().y;
   
    float testColor = dot(frac(In.UV.xy * 10) ,0.5.xx) + 0.25;
   
    return float4(EncUnitRangeToRGBA8(linearDepth).xyz, testColor);
}

float4 DecodeBake(float2 uv)
{
    float4 sample = tex2D(BakeSampler, uv);
    float linearZ = DecUnitRangeToRGBA8(float4(sample.xyz,0));

    // back to viewspace...
    float2 nearFar = GetPlanesFromDxProjection();
    linearZ = linearZ * (nearFar.y-nearFar.x) + nearFar.x;

#define VIEWTOUVMULT (float2(Proj[0][0], -Proj[1][1]) * 0.5)
#define VIEWTOUVADD float2(0.5, 0.5)   
    float2 xy = ((uv - VIEWTOUVADD) * linearZ)/VIEWTOUVMULT;
   
    return float4(xy, linearZ, sample.w); // view xyz, pattern
}

float4 CopyPS(FSQuadVS_InOut In) : COLOR
{
    return float4(DecodeBake(In.UV).zzz / 5, 1);
    return float4(DecodeBake(In.UV).www, 1);
}

float3 ReprojectView(float3 viewPos)
{
    // Here we should go from the view position of the baked eye to the one of the other
    // it can be pretty generic but it assumes the difference will be only along the
    // screen X axis...
   
    return viewPos + float3(0.2,0,0);
}

float4 ReprojectPS(FSQuadVS_InOut In) : COLOR
{
    float reprojDepth = 9999;
    float reprojColor = 0;
   
    float nearestColor = 0;
    float nearestDistance = 9999;
   
    float4 decodedCenter = DecodeBake(In.UV); // xyz = view, w = test pattern
   
    float4 projectedCenter = mul(float4(decodedCenter.xyz,1), Proj);
    projectedCenter /= projectedCenter.w;
   
    for(float i=-63; i<64; i++) // note: brute force = bad, just a test
    {
        float2 sampleUV = In.UV + float2(i / BufferSize.x, 0);
       
        // we assume in this test that the "color" is baked in w
        // also, we assume that the scene fills the entire screen,
        // we draw something everywhere...
        float4 decoded = DecodeBake(sampleUV);   
       
        float3 newView = ReprojectView(decoded.xyz);
       
        float4 projectedNewView = mul(float4(newView,1), Proj);
        projectedNewView /= projectedNewView.w;
       
        float rowDist = abs(projectedNewView.x - projectedCenter.x);
       
        // Check if the new sample covers our pixel
        // note: al these tests can be made "fuzzy" by constructing
        // a weighting function instead
        if(rowDist < (2.0 / BufferSize.x))
        {
            if(reprojDepth>newView.z)
            {
                reprojColor = decoded.w;
                reprojDepth = newView.z;
            }
        }
       
        // We'll use this to fill the holes if any
        // note: this is a very crude inpaiting method, something
        // way better should be used for the method to work decently
        float heuristicDistance = rowDist*rowDist / newView.z;
        if(heuristicDistance < nearestDistance)
        {
            nearestColor = decoded.w;
            nearestDistance = heuristicDistance;
        }
    }
   
    // If we didn't find anything, we pull something from thin air...
    if(reprojColor==0) reprojColor = nearestColor;

    //return float4(decodedCenter.www,1);
    //return float4(reprojColor.xxx,1);
    return float4(reprojColor,decodedCenter.ww,1);
}

// -- Techniques

technique Main
<
    string Script =
        "Pass=Bake;"
        "Pass=FullScreen;"
    ;
>
{
    pass Bake
    <
        string Script =
            "RenderColorTarget0=Bake;"
            "ClearSetColor=ClearParam;"
            "Clear=Color;"
            "Clear=Depth;"
            "Draw=Geometry;";
    >
    {
        ZEnable = true;
        ZWriteEnable = true;
       
        VertexShader = compile vs_3_0 GeomVS();
        PixelShader = compile ps_3_0 BakePS();
    }
   
    pass FullScreen
    <
        string Script =
            "RenderColorTarget0=;"
            "Draw=Buffer;";
    >
    {
        ZEnable = false;
        ZWriteEnable = false;
   
        VertexShader = compile vs_3_0 FSQuadVS();
        PixelShader = compile ps_3_0 ReprojectPS(); //CopyPS();
    }
};