Cel Shading in Torque - An Evolutionary Approach

Since there's been some interest in how I've been doing cel shading, I'm providing some examples of what I've done. And to provide some context, I'm going to show how I "evolved" what I'm using now. This first post explains the simplest (and first) cel shading I got working.

The First Attempt - Flat Cel Shading

This method makes great screen shots, and doesn't look too shabby in the game either. But since it "flat" shades entire polygons according to the cel shading manner, it can look rather "old school" (or just "old tech").

NOTE: For simplicity, I'm just hard-coding the light vector, and the cel shading band calculations. And I have no idea if this works in DirectX. I haven't tried it.

In ts\tsMesh.cc, you need to change TSMesh::render() to this (new code highlighted, maybe):

void TSMesh::render(S32 frame, S32 matFrame, TSMaterialList * materials, const MatrixF *objectTransform)
{
   if( vertsPerFrame <= 0 ) {
      return;
   }

   S32 firstVert  = vertsPerFrame * frame;
   S32 firstTVert = vertsPerFrame * matFrame;

   if (getFlags(Billboard))
   {
      if (getFlags(BillboardZAxis))
         forceFaceCameraZAxis();
      else
         forceFaceCamera();
   }

   const Point3F * normals = getNormals(firstVert);
   saveMergeNormals(); // verts & tverts saved and restored on tsshapeinstance::setStatics

   [b]// drm - cel shading setup
   VectorF lightVector(1,1,1);[/b]

   // set up vertex arrays -- already enabled in TSShapeInstance::render
   glVertexPointer(3,GL_FLOAT,0,&verts[firstVert]);
   glNormalPointer(GL_FLOAT,0,normals);
   glTexCoordPointer(2,GL_FLOAT,0,&tverts[firstTVert]);
   if (TSShapeInstance::smRenderData.detailMapMethod == TSShapeInstance::DETAIL_MAP_MULTI_1 ||
       TSShapeInstance::smRenderData.detailMapMethod == TSShapeInstance::DETAIL_MAP_MULTI_2)
   {
      glClientActiveTextureARB(GL_TEXTURE0_ARB + TSShapeInstance::smRenderData.detailMapTE);
      glTexCoordPointer(2,GL_FLOAT,0,&tverts[firstTVert]);
      glClientActiveTextureARB(GL_TEXTURE0_ARB + TSShapeInstance::smRenderData.baseTE);
   }

   // lock...
   bool lockArrays = dglDoesSupportCompiledVertexArray();
   if (lockArrays)
      glLockArraysEXT(0,vertsPerFrame);

   [b]// drm - disable lighting
   bool oldlighting=glIsEnabled(GL_LIGHTING);
   glDisable(GL_LIGHTING);[/b]

   for (S32 i=0; i<primitives.size(); i++)
   {
      TSDrawPrimitive & draw = primitives[i];
      AssertFatal(draw.matIndex & TSDrawPrimitive::Indexed,
                  "TSMesh::render: rendering of non-indexed meshes no longer supported");

      // material change?
      if ( ((TSShapeInstance::smRenderData.materialIndex ^ draw.matIndex) &
            (TSDrawPrimitive::MaterialMask|TSDrawPrimitive::NoMaterial)) != 0)
         setMaterial(draw.matIndex,materials);

      [b]// drm - cel shading
      // "average" normals for the surface
      Point3F tmpNormal(0,0,0);
      for (S32 ee=0; ee<draw.numElements; ee++)
         tmpNormal+=normals[indices[draw.start+ee]-firstVert];
      tmpNormal.normalize();
      // dot product of the vertex normal and light
      F32 dotP = mDot(tmpNormal,lightVector);
      // adjust to cel shading
      F32 shade;
      if (dotP<=-0.50)
         shade=0.35;
      else if (dotP<=0.00)
         shade=0.45;
      else if (dotP<=0.45)
         shade=0.65;
      else
         shade=1.0;
      // set the color for all vertices in this primitive
      glColor3f(shade,shade,shade);
      //[/b]
      S32 drawType = getDrawType(draw.matIndex>>30);
      glDrawElements(drawType,draw.numElements,GL_UNSIGNED_SHORT,&indices[draw.start]);
   }

   // drm
   if (oldlighting) glEnable(GL_LIGHTING);

   // unlock...
   if (lockArrays)
      glUnlockArraysEXT();

   restoreMergeNormals();
}

(more in next post)

-David