Game Development Community

Outline Shader Broken ? (1.1b1)

by Cai Yundong · in Torque 3D Professional · 05/12/2010 (10:56 pm) · 6 replies

http://www.torquepowered.com/community/forums/viewthread/93601/
http://www.torquepowered.com/community/forums/viewthread/97401

Based on the Above 2 Threads, I got the Outline Shader Working All the way till 1.1 alpha. I Just started porting my project to 1.1b1 and discovered i could not get it to work. Is anyone experiencing the same problem ?

Could it be due to the AA problem mentioned in the below thread ?
http://www.torquepowered.com/community/forums/viewthread/114961

#1
06/01/2010 (6:06 am)
logged key: TQA-188
#2
08/19/2010 (11:23 pm)
Unfortunately we don't have the resources to delve deeply into a community maintained effect.

If someone with more knowledge of this PostEffect can identify a direct bug in the engine that is keeping it from working then we will definitely look into it.

Until then, I would recommend looking at the EdgeAA PosteEffect since it is essentially an outline shader and it is currently working.
#3
08/20/2010 (2:29 am)
Tested with T3D 1.1 Beta 2

On core/scripts/client/postfx/PostFX.cs:
function initPostEffects()
{
   ....

   if($pref::PostEffect::HDR)   
      HDRPostFX.enable();
      
   if($pref::PostEffect::Outliner)   // ADDED
      OutlinerPostFX.enable();       // ADDED

   ....
}

On core/scripts/client/postfx/Outliner.cs:
singleton GFXStateBlockData( PFX_DefaultOutlineStateBlock )
{
   zDefined = true;
   zEnable = false;
   zWriteEnable = false;
      
   samplersDefined = true;
   samplerStates[0] = SamplerClampLinear;
};

singleton ShaderData( PFX_OutlineShader )
{   
   DXVertexShaderFile 	= "shaders/common/postFx/postFxV.hlsl";
   DXPixelShaderFile 	= "shaders/common/postFx/outlineShaderP.hlsl";
         
   samplerNames[0] = "$inputTex";
   
   pixVersion = 3.0;
};

singleton ShaderData( PFX_OutlineEdgeDetectShader )
{   
   DXVertexShaderFile 	= "shaders/common/postFx/postFxV.hlsl";
   DXPixelShaderFile 	= "shaders/common/postFx/edgeaa/edgeDetectP.hlsl";
 
   samplerNames[0] = "$inputTex";
   
   pixVersion = 3.0;
};

singleton PostEffect( OutlinerPostFX )
{
   renderTime = "PFXAfterDiffuse";
      
   shader = PFX_OutlineEdgeDetectShader;
   stateBlock = PFX_DefaultOutlineStateBlock;
   texture[0] = "#prepass";
   target = "$outTex";
   
   new PostEffect()
   {
      shader = PFX_OutlineShader;
      stateBlock = PFX_DefaultOutlineStateBlock;
      texture[0] = "$inTex"; 
      texture[1] = "$backBuffer";
      target = "$backBuffer";
   };
};

The PixelShader on shaders/common/postFX/outlineDetectP.hlsl
#include "postFx.hlsl"
#include "shadergen:/autogenConditioners.h"

// GPU Gems 3, pg 443-444
float GetEdgeWeight(float2 uv0, in sampler2D prepassBuffer, in float2 targetSize)
{
   float2 offsets[9] = {
      float2( 0.0,  0.0),
      float2(-1.0, -1.0),
      float2( 0.0, -1.0),
      float2( 1.0, -1.0),
      float2( 1.0,  0.0),
      float2( 1.0,  1.0),
      float2( 0.0,  1.0),
      float2(-1.0,  1.0),
      float2(-1.0,  0.0),
   };
   
   
   float2 PixelSize = 1.0 / targetSize;
   
   float Depth[9];
   float3 Normal[9];
   
   for(int i = 0; i < 9; i++)
   {
      float2 uv = uv0 + offsets[i] * PixelSize;
      float4 gbSample = prepassUncondition( tex2D(prepassBuffer, uv) );
      Depth[i] = gbSample.a;
      Normal[i] = gbSample.rgb;
   }
   
   float4 Deltas1 = float4(Depth[1], Depth[2], Depth[3], Depth[4]);
   float4 Deltas2 = float4(Depth[5], Depth[6], Depth[7], Depth[8]);
   
   Deltas1 = abs(Deltas1 - Depth[0]);
   Deltas2 = abs(Depth[0] - Deltas2);
   
   float4 maxDeltas = max(Deltas1, Deltas2);
   float4 minDeltas = max(min(Deltas1, Deltas2), 0.00001);
   
   float4 depthResults = step(minDeltas * 25.0, maxDeltas);
   
   Deltas1.x = dot(Normal[1], Normal[0]);
   Deltas1.y = dot(Normal[2], Normal[0]);
   Deltas1.z = dot(Normal[3], Normal[0]);
   Deltas1.w = dot(Normal[4], Normal[0]);
   
   Deltas2.x = dot(Normal[5], Normal[0]);
   Deltas2.y = dot(Normal[6], Normal[0]);
   Deltas2.z = dot(Normal[7], Normal[0]);
   Deltas2.w = dot(Normal[8], Normal[0]);
   
   Deltas1 = abs(Deltas1 - Deltas2);
   
   float4 normalResults = step(0.4, Deltas1);
   
   normalResults = max(normalResults, depthResults);
   
   return dot(normalResults, float4(1.0, 1.0, 1.0, 1.0)) * 0.25;
}

float4 main( PFXVertToPix IN, 
             uniform sampler2D prepassBuffer :register(S0),
             uniform float2 targetSize : register(C0) ) : COLOR0
{
   return GetEdgeWeight(IN.uv0, prepassBuffer, targetSize );
}

The VertexShader on shaders/common/postFX/outlineDetectV.hlsl
#include "postFx.hlsl"
#include "shadergen:/autogenConditioners.h"

float4 main( PFXVertToPix IN, 
             uniform sampler2D edgeBuffer :register(S0), uniform sampler2D backBuffer : register(S1) ) : COLOR0
{
   float4 e = float4( tex2D( edgeBuffer, IN.uv0 ).rrr, 1.0 );
   e.r = e.g = e.b = 1.0 - e.r;
   return tex2D(backBuffer, IN.uv0) * e;
}

The unique issue regarding the overlapping with the forest mesh:

img710.imageshack.us/img710/3923/screenshot00600000.th.png
#4
08/20/2010 (4:07 am)
@Alfio - Check the materials on those trunks... i suspect they are alpha blended and because of that are forward rendered and are not in the depth buffer.
#5
08/20/2010 (8:50 am)
You're absolutely right, sorry for the stupid mistake

img199.imageshack.us/img199/7684/screenshot00700003.th.png
#6
08/20/2010 (5:54 pm)
Moved to General Discussion.