DepthBuffer sampling for DoF & SSAO Shader

#1 renderInstance/renderDepthMgr.cpp

#include "renderDepthMgr.h"
#include "materials/sceneData.h"
#include "sceneGraph/sceneGraph.h"
#include "materials/matInstance.h"
#include SHADER_CONSTANT_INCLUDE_FILE
#include "T3D/gameTSCtrl.h"
#include "gfx/primBuilder.h"


//**************************************************************************
// RenderDepthMgr
//**************************************************************************
void RenderDepthMgr::setupSGData( RenderInst *ri, SceneGraphData &data )
{
   dMemset( &data, 0, sizeof( SceneGraphData ) );
   data.camPos = gRenderInstManager.getCamPos();
   data.objTrans = *ri->objXform;
}

//-----------------------------------------------------------------------------
// render
//-----------------------------------------------------------------------------
void RenderDepthMgr::render()
{
   mDepth.set( 256, 256, GFXFormatR8G8B8A8, &GFXDefaultRenderTargetProfile, 1 );
   GFX->pushWorldMatrix();


   
   // set render states
   if( gClientSceneGraph->isReflectPass() )
   {
      GFX->setCullMode( GFXCullCW );
   }
   else
   {
      GFX->setCullMode( GFXCullCCW );
   }

   for( U32 i=0; i<TEXTURE_STAGE_COUNT; i++ )
   {
      GFX->setTextureStageAddressModeU( i, GFXAddressWrap );
      GFX->setTextureStageAddressModeV( i, GFXAddressWrap );

      GFX->setTextureStageMagFilter( i, GFXTextureFilterLinear );
      GFX->setTextureStageMinFilter( i, GFXTextureFilterLinear );
      GFX->setTextureStageMipFilter( i, GFXTextureFilterLinear );
   }


   // init loop data
   GFXVertexBuffer * lastVB = NULL;
   GFXPrimitiveBuffer * lastPB = NULL;
   SceneGraphData sgData;
   U32 binSize = mElementList.size();
   GFX->pushActiveRenderTarget();
   GFX->setActiveRenderTarget( mDepth );
   GFX->clear( GFXClearTarget, ColorI(0.5,0.5,0.5,0.5), 1.0f, 0 );
   for( U32 j=0; j<binSize; )
   {
      RenderInst *ri = mElementList[j].inst;

      setupSGData( ri, sgData );
      MatInstance mat("DepthMaterial");

      U32 matListEnd = j;
      //bool firstmatpass = true;
      while(mat.setupPass( sgData ) )
      {
         U32 a;
         for( a=j; a<binSize; a++ )
         {
            RenderInst *passRI = mElementList[a].inst;

			   // don't break the material multipass rendering...
			   /*if(firstmatpass)
			   {
				   if(passRI->primitiveFirstPass)
				   {
					   bool &firstpass = *passRI->primitiveFirstPass;
					   if(firstpass)
					   {
					   }
					   else
					   {
						   GFX->setAlphaBlendEnable(false);
                           GFX->setSrcBlend(GFXBlendSrcAlpha);
						   GFX->setDestBlend(GFXBlendOne);
						   //GFX->setDestBlend(GFXBlendZero);
						   //continue;
					   }
				   }
				   else
				   {
					   GFX->setAlphaBlendEnable(false);
					   GFX->setSrcBlend(GFXBlendOne);
					   GFX->setDestBlend(GFXBlendZero);
				   }
			   }*/


            // fill in shader constants that change per draw
            //-----------------------------------------------
            GFX->setVertexShaderConstF( 0, (float*)passRI->worldXform, 4 );

            // set object transform
            MatrixF objTrans = *passRI->objXform;
            objTrans.transpose();
            GFX->setVertexShaderConstF( VC_OBJ_TRANS, (float*)&objTrans, 4 );
            objTrans.transpose();
            objTrans.inverse();

            // fill in eye data
            Point3F eyePos = gRenderInstManager.getCamPos();
            objTrans.mulP( eyePos );
            GFX->setVertexShaderConstF( VC_EYE_POS, (float*)&eyePos, 1 );

            // set buffers if changed
            if( lastVB != passRI->vertBuff->getPointer() )
            {
               GFX->setVertexBuffer( *passRI->vertBuff );
               lastVB = passRI->vertBuff->getPointer();
            }
            if( lastPB != passRI->primBuff->getPointer() )
            {
               GFX->setPrimitiveBuffer( *passRI->primBuff );
               lastPB = passRI->primBuff->getPointer();
            }

            // draw it
            GFX->drawPrimitive( passRI->primBuffIndex );
         }

         matListEnd = a;
		   //firstmatpass = false;
      }

      // force increment if none happened, otherwise go to end of batch
      j = ( j == matListEnd ) ? j+1 : matListEnd;

   }
   GFX->popActiveRenderTarget();
   GameTSCtrl *tsCtrl = dynamic_cast<GameTSCtrl*>( Sim::findObject("PlayGui") );
   if(tsCtrl)
   {
   RectI vp = GFX->getViewport();
   //tsCtrl->renderTestEffect(vp,NULL,GFXFillSolid,mDepth);
   }
   GFX->popWorldMatrix();
}

#2 renderInstance/renderDepthMgr.h

//-----------------------------------------------------------------------------
// Torque Game Engine Advanced
// Copyright (C) GarageGames.com, Inc.
//-----------------------------------------------------------------------------
#ifndef _RENDER_DEPTH_MGR_H_
#define _RENDER_DEPTH_MGR_H_

#include "renderElemMgr.h"

//**************************************************************************
// RenderDepthMgr
//**************************************************************************
class RenderDepthMgr : public RenderElemMgr
{
private:
   bool mFirstRun;
   ShaderData      * mDepthShader;
   ShaderData      * mEffectShader;
   GFXTarget       * mDepth;
   void setupSGData( RenderInst *ri, SceneGraphData &data );

public:
   virtual void render();

   //void copyToScreen( RectI &viewport );
};


#endif

If anyone could help us out with this, we would really appreciate it.

#3 If someone gets it to compile, to test it just uncomment:

tsCtrl->renderTestEffect(vp,NULL,GFXFillSolid,mDepth);

in renderDepthMgr.cpp (4th like from the bottom). And in T3D/gameTSCtrl.cpp add:

void GameTSCtrl::renderTestEffect(const RectI &updateRect, U32 objMask, GFXFillMode fMode, GFXTexHandle mTex)
{

   FrameAllocator::setWaterMark(0);

   Point2I a = updateRect.point;
   Point2I b = updateRect.point + updateRect.extent;


   GFX->setZEnable( true ); //glEnable(GL_DEPTH_TEST);
   GFX->setZFunc( GFXCmpLessEqual ); //glDepthFunc(GL_LEQUAL);



   // Need to consoldate to one clear call // glClear(GL_DEPTH_BUFFER_BIT);
   GFX->setCullMode( GFXCullNone );//glDisable(GL_CULL_FACE);




   //************************************
   //************************************
   //************************************
   GFX->setFillMode(fMode);


	// set the clip
    GFX->setClipRect(updateRect);

	// turn off  z clipping
	GFX->setZEnable( false );

   //************************************
   //************************************
   //************************************


//after this is all surfaces
//**************************************

    // set ortho projection matrix
	// this makes ur view of the effect appears as directly above
    MatrixF proj = GFX->getProjectionMatrix();
    MatrixF newMat(true);
    GFX->setProjectionMatrix( newMat );
    GFX->pushWorldMatrix();
    GFX->setWorldMatrix( newMat );   
    GFX->setVertexShaderConstF( 0, (float*)&newMat, 4 );  // send the matrix to the shader
      
// 
//**********************************

	// ortho geometry
   	GFXVertexBuffer *vb=NULL;
	//PrimBuild::color4f( mCurColor.red , mCurColor.green , mCurColor.blue, mCurColor.alpha );
    PrimBuild::beginToBuffer( GFXTriangleFan, 4 );
      PrimBuild::texCoord2f( 0.0, 1.0 );
      PrimBuild::vertex3f( -1.0 , -1.0 , 0.0 );
      
      PrimBuild::texCoord2f( 0.0, 0.0 );
      PrimBuild::vertex3f( -1.0 ,  1.0 , 0.0 );
      
      PrimBuild::texCoord2f( 1.0, 0.0 );
      PrimBuild::vertex3f(  1.0 ,  1.0 , 0.0 );
      
      PrimBuild::texCoord2f( 1.0, 1.0 );
      PrimBuild::vertex3f(  1.0 , -1.0 , 0.0 );
    U32 numPrims;
    vb =  PrimBuild::endToBuffer( numPrims );
   

   //mDoFEffectShader->shader->process();

   GFX->setTexture( 0, mTex );

   GFX->setTexture(1, mTex);

   GFX->setVertexBuffer( vb );
   GFX->drawPrimitive( GFXTriangleFan, 0, 2 );

   GFX->disableShaders();


//**********************************************


   // render state cleanup
   GFX->setAlphaBlendEnable( false );
   GFX->setCullMode( GFXCullNone );
   GFX->setLightingEnable( false );
   GFX->setZWriteEnable( true );
   GFX->setTextureStageColorOp( 0, GFXTOPDisable );  
   GFX->disableShaders();

   GFX->popWorldMatrix();


	FrameAllocator::setWaterMark(0);

}

#4 MDepth is a pointer. Just like the error message says, you must use an "->" to dereference a pointer. No offense intended, but are you new at C++? You may want to brush up on pointer usage before tackling engine changes.

Anyway, this will compile if you do what the compiler says and change that line to mDepth->set instead of mDepth.set

But I don't see anywhere that mDepth (a private variable) is assigned a value. The magic eight ball says: "You have an access violation in your near future".

#5 I am new to C++ but I have tried that and it gives me this error:

renderDepthMgr.cpp
..\..\..\..\..\engine\source\renderInstance\renderDepthMgr.cpp(32) : error C2039: 'set' : is not a member of 'GFXTarget'
e:\HotB\HotB_Alpha (0.10)\engine\source\gfx/gfxTarget.h(24) : see declaration of 'GFXTarget'

#6 That error means the function you are wanting to call doesn't exist.

Here's the real problem: Even if it did exist, the pointer is invalid. It has no value, and will crash your program when that function is called.

What it looks like you may want to be doing is rendering to a texture. If so, you would need to create a GFXTextureTarget instead of a GFXTarget, and assign it a texture.

You may want to get the person who gave you this code to look at it, as it is clearly unfinished.

#7 Before 1.7 it was working. The only problem was it dropped the fps quite a bit, but it wasnt optimized.

In renderDepthMgr.h

GFXTarget * mDepth; was GFXTexHandle mDepth;

In renderDepthMgr.cpp

GFX->pushActiveRenderTarget(); was GFX->pushActiveRenderSurfaces();
GFX->popActiveRenderTarget(); was GFX->popActiveRenderSurfaces();
GFX->setActiveRenderTarget(); was GFX->setActiveRenderSurfaces();

But now with TGEA 1.7 setActiveRenderSurfaces, popActiveRenderSurfaces and pushActiveRenderSurfaces dont exist anymore. You would get the error:

..\..\..\..\..\engine\source\renderInstance\renderDepthMgr.cpp(64) : error C2664: 'GFXDevice::setActiveRenderTarget' : cannot convert parameter 1 from 'GFXTexHandle' to 'GFXTarget *'

The changes that were made might be what is wrong, but prior to 1.7 it WAS working.

#8 @Adam - Render targets work differently in TGEA. Do a search thru the code for setActiveRenderTarget and you'll find an example of how it use it now.