max2dtsExporterPro enhancement: user-specified vertex normals!

This weekend was a full of fun - dealing with MAX SDK and exporters. I've never worked with max sdk, so it was a good learning time for me..

So, what it is about:

if you have a model with separate meshes and want to look them smooth, or you want to specify/edit normals manually (via "Edit Normals" modifier) - you will get disappointed after export - by default max does not give normals information, so mesh->getNormal(idx) will return garbage, or some random 0-1 values (if run maxMesh->buildNormals()). Yes, you can play with smooth groups, but... you know, if you have complex geo like in character model, this will not help :(

Luckily, since MAX SDK 7.0 we have a new class: MeshNormalSpec -- this is especially for our needs.

Here are the pictures.
Description:
1: Create cylinder + Convert to editable mesh.
2: Change geo a bit
3: Split
4: Detach part (make separate mesh)
5: Notice the lighting is bad now due to the normals now different on vertices
6: as normals got a different route - fix it! (sepect both meshes -> Modifier List- > Edit Normals, select needed normals and click to "Average > Selected")
7: Collapse (convert back to editable mesh)
..: [ add skin if needed, animate, etc ]
8: Export: Bad Result :(
This is because exporter uses smooth groups for storing normals' information, not the actually specified normals.

Now, with the changes:
8: good result, as expected!

P.S. If no custom normals specified, exporter will use SmoothGroups for normals calculation (old behaviour)
P.P.S. Please remember: when converting from poly or patch to mesh - max looses all custom normals information, so fix normals only when you have finished most of the modeling)

Changes to max2dtsExporterPro (since the public release of Pro version, I've never used old exporter, so don't know it it's possible to implement these features there):

Open tools/max2dtsExporterPro/maxAppMesh.cpp
Add at top of file (in max includes section):

#include <MeshNormalSpec.h>

Changes in bold and wrapped with // vertex normals fix

AppMeshLock MaxAppMesh::lockMesh(const AppTime & time, const Matrix<4,4,F32> & objectOffset)
   {
      ...
      TriObject * tri = getTriObject(mMaxNode,SecToTicks(time.getF32()),delTri);
      ::Mesh & maxMesh = tri->mesh;

      S32 i;
      S32 lastMatIdx = -1;
      ...
      mSmooth.clear();
      mVertId.clear();
      [b]// vertex normals fix // begin
      MeshNormalSpec * nrm = maxMesh.GetSpecifiedNormals(); // vertex normals fix
      // Create transform based on objectOffset for transforming normals
      Matrix<4,4,F32> normOffset = objectOffset;
      normOffset[0][3] = 0.0f;
      normOffset[1][3] = 0.0f;
      normOffset[2][3] = 0.0f;
      normOffset[3][0] = 0.0f;
      normOffset[3][1] = 0.0f;
      normOffset[3][2] = 0.0f;
      normOffset[3][3] = 1.0f;
      F32 normScale = 0.0f;
      for (i=0; i<9; i++)
      {
         S32 j = i % 3;
         normScale += normOffset[(i-j)/3][j];
      }
      normScale /= 3.0f;
      normScale = 1.0f/normScale;
      if(nrm && nrm->GetNumNormals())
         AppConfig::PrintDump(PDObjectOffsets,avar("Using custom normals!\r\n"));
      else
         AppConfig::PrintDump(PDObjectOffsets,avar("Using normals info from smooth groups!\r\n"));
      // vertex normals fix // end[/b]
      // start out with faces and crop data allocated
      //mFaces.resize(maxMesh.getNumFaces());
      ...
            tvert2.x(tv2.x);
            tvert2.y(1.0f-tv2.y);
            [b]// vertex normals fix // begin
            if(nrm && nrm->GetNumNormals())
            {
               // compute normal (TODO: handle non-uniform scale properly)
               Point3 maxNorm;
               F32 norm0, norm1, norm2;
               // Now we need to get normals for all three idxX
               int normID;

               /// First
               normID = nrm->Face(i).GetNormalID(0);
               maxNorm = nrm->Normal(normID).Normalize();
               //maxNorm = maxMesh.getNormal(idx0);
               norm0 = F32(maxNorm.x);
               norm1 = F32(maxNorm.y);
               norm2 = F32(maxNorm.z);
               if (mirror)
               {
                  F32 tmp = norm1;
                  norm1 = norm2;
                  norm2 = tmp;
               }
               normx0 = Point3D(norm0, norm1, norm2);
               normx0 = normOffset * normx0;
               normx0 *= normScale;
               if (normx0.length() > 0.001f || normx0.length() < 0.001f)
                  normx0.normalize();
               else
                  normx0 = Point3D(0.f,0.f,1.f);
               //AppConfig::PrintDump(PDObjectOffsets,avar("Parsing 0 vertex %d: %+0.3f %+0.3f %+0.3f || %+0.3f %+0.3f %+0.3f\r\n", idx0, normx0.x(), normx0.y(), normx0.z(), norm0, norm1, norm2));
               /// Second
               normID = nrm->Face(i).GetNormalID(2);
               maxNorm = nrm->Normal(normID).Normalize();
               //maxNorm = maxMesh.getNormal(idx1);
               norm0 = F32(maxNorm.x);
               norm1 = F32(maxNorm.y);
               norm2 = F32(maxNorm.z);
               if (mirror)
               {
                  F32 tmp = norm1;
                  norm1 = norm2;
                  norm2 = tmp;
               }
               normx1 = Point3D(norm0, norm1, norm2);
               normx1 = normOffset * normx1;
               normx1 *= normScale;
               if (normx1.length() > 0.001f || normx1.length() < 0.001f)
                  normx1.normalize();
               else
                  normx1 = Point3D(0.f,0.f,1.f);
               //AppConfig::PrintDump(PDObjectOffsets,avar("Parsing 1 vertex %d: %+0.3f %+0.3f %+0.3f || %+0.3f %+0.3f %+0.3f\r\n", idx1, normx1.x(), normx1.y(), normx1.z(), norm0, norm1, norm2));
               // Third
               normID = nrm->Face(i).GetNormalID(1);
               maxNorm = nrm->Normal(normID).Normalize();
               //maxNorm = maxMesh.getNormal(idx2);
               norm0 = F32(maxNorm.x);
               norm1 = F32(maxNorm.y);
               norm2 = F32(maxNorm.z);
               if (mirror)
               {
                  F32 tmp = norm1;
                  norm1 = norm2;
                  norm2 = tmp;
               }
               normx2 = Point3D(norm0, norm1, norm2);
               normx2 = normOffset * normx2;
               normx2 *= normScale;
               if (normx2.length() > 0.001f || normx2.length() < 0.001f)
                  normx2.normalize();
               else
                  normx2 = Point3D(0.f,0.f,1.f);
               //AppConfig::PrintDump(PDObjectOffsets,avar("Parsing 2 vertex %d: %+0.3f %+0.3f %+0.3f || %+0.3f %+0.3f %+0.3f\r\n", idx2, normx2.x(), normx2.y(), normx2.z(), norm0, norm1, norm2));
            }
            // vertex normals fix // end[/b]
         }
         tsFace.type |= maxFace.getMatID();  // return max mat id...gets converted to ts mat id by shapemimic

			mFaces.push_back(tsFace);

         // now add indices...this is easy right now...later we'll mess this up
         [b]// vertex normals fix // begin
         if (nrm && nrm->GetNumNormals())
         {
            mIndices.push_back(addVertex(vert0, normx0, tvert0, idx0));
            mIndices.push_back(addVertex(vert1, normx1, tvert1, idx1));
            mIndices.push_back(addVertex(vert2, normx2, tvert2, idx2));
         }
         else
         {
            mIndices.push_back(addVertex(vert0,tvert0,idx0,maxFace.smGroup));
            mIndices.push_back(addVertex(vert1,tvert1,idx1,maxFace.smGroup));
            mIndices.push_back(addVertex(vert2,tvert2,idx2,maxFace.smGroup));
         }
         // vertex normals fix // end[/b]

The logic is based on Maya2dts exporter and so far it seems to work fine for me.

Let me know if there is something "wrong" with it or if you find bug - let's make a good exporter!