torquescript math routines for large integers

#1 So you got me thinking about this...

I found that Torque uses a 64-bit float table to hold those numbers. It seems that if you remove the code that converts from a float to scientific notation you can get very good precision at high numbers. I'm still working on this but it appears to be a start.

A few notes, it appears you cannot use a number larger than 2147483647 (2^31-1) (the limit of S32) on each side of an operator although that might be fixable. Also if you store the value into a variable it goes into scientific notation, also probably fixable. The in game editors and other places still use %g with scanf so that might break as well. Also only tested on Windows so far.

echo(2147483647 + 2147483647 + 2147483647 + 2147483647 + 2147483647 + 2147483647 + 2147483647 + 2147483647 + 1);
==>17179869177

To remove the scientific notation change the setFloatValue function in stringStack.h to (around line 80 in 1.5.1):

void setFloatValue(F64 v)
{
   validateBufferSize(mStart + 32);
   dSprintf(mBuffer + mStart, 32, "%.7f", v);

   U32 len = dStrlen(mBuffer + mStart);
   char *string = mBuffer + mStart;
   while (string[len-1] == '0')
      len--;

   if (string[len-1] == '.')
      len--;

   string[len] = 0;
   mLen = len;
}

#2 hey Wes! that's very cool! this issue has been floated (ho ho) numerous times and it's always looked like a major pita to do anything about it, but your change seems very straight-forward. i'll give it a shot monday.

#3 I decided to add support for storing the 64 bit float as a script variable and added a TypeF64 to expose F64 variables to script for C++ classes. This time I had to change a few files so it's easier to give out a patch. This includes the fix from above.

Still suffers the 2^31-1 limit on either side of operators, I will look into that next. Also still only tested on Windows so far.

Note: Hit a 1000 word limit. So here is part 1.

Index: engine/console/compiledEval.cc
===================================================================
--- engine/console/compiledEval.cc	(revision 2053)
+++ engine/console/compiledEval.cc	(working copy)
@@ -97,9 +97,18 @@
 
    char *getFloatArg(F64 arg)
    {
+      char *ret = STR.getArgBuffer(32);
+		dSprintf(ret, 32, "%.7f", arg);
 
-      char *ret = STR.getArgBuffer(32);
-      dSprintf(ret, 32, "%g", arg);
+		U32 len = dStrlen(ret);
+		while (ret[len-1] == '0')
+			len--;
+
+		if (ret[len-1] == '.')
+			len--;
+
+		ret[len] = 0;
+
       return ret;
    }
 
Index: engine/console/compiler.cc
===================================================================
--- engine/console/compiler.cc	(revision 2053)
+++ engine/console/compiler.cc	(working copy)
@@ -163,7 +163,16 @@
 
 U32 CompilerStringTable::addFloatString(F64 value)
 {
-   dSprintf(buf, sizeof(buf), "%g", value);
+	dSprintf(buf, sizeof(buf), "%.7f", value);
+
+	U32 len = dStrlen(buf);
+	while (buf[len-1] == '0')
+		len--;
+
+	if (buf[len-1] == '.')
+		len--;
+
+	buf[len] = 0;
    return add(buf);
 }
 void CompilerStringTable::reset()
Index: engine/console/consoleInternal.h
===================================================================
--- engine/console/consoleInternal.h	(revision 2053)
+++ engine/console/consoleInternal.h	(working copy)
@@ -152,7 +152,7 @@
         S32 type;
         char *sval;
         U32 ival;  // doubles as strlen when type = -1
-        F32 fval;
+        F64 fval;
         U32 bufferLen;
         void *dataPtr;
 
@@ -166,19 +166,19 @@
             else
                 return dAtoi(Con::getData(type, dataPtr, 0));
         }
-        F32 getFloatValue()
+        F64 getFloatValue()
         {
             if(type <= TypeInternalString)
                 return fval;
             else
-                return dAtof(Con::getData(type, dataPtr, 0));
+                return dAtof64(Con::getData(type, dataPtr, 0));
         }
         const char *getStringValue()
         {
             if(type == TypeInternalString)
                 return sval;
             if(type == TypeInternalFloat)
-                return Con::getData(TypeF32, &fval, 0);
+                return Con::getData(TypeF64, &fval, 0);
             else if(type == TypeInternalInt)
                 return Con::getData(TypeS32, &ival, 0);
             else
@@ -188,7 +188,7 @@
         {
             if(type <= TypeInternalString)
             {
-                fval = (F32)val;
+                fval = static_cast<F64>(val);
                 ival = val;
                 if(sval != typeValueEmpty)
                 {
@@ -204,7 +204,7 @@
                 Con::setData(type, dataPtr, 0, 1, &dptr);
             }
         }
-        void setFloatValue(F32 val)
+        void setFloatValue(F64 val)
         {
             if(type <= TypeInternalString)
             {
@@ -220,7 +220,7 @@
             }
             else
             {
-                const char *dptr = Con::getData(TypeF32, &val, 0);
+                const char *dptr = Con::getData(TypeF64, &val, 0);
                 Con::setData(type, dataPtr, 0, 1, &dptr);
             }
         }

#4 Part 2.

Index: engine/console/consoleTypes.cc
===================================================================
--- engine/console/consoleTypes.cc	(revision 2053)
+++ engine/console/consoleTypes.cc	(working copy)
@@ -279,6 +279,35 @@
 }
 
 //////////////////////////////////////////////////////////////////////////
+// TypeF64
+//////////////////////////////////////////////////////////////////////////
+ConsoleType( float, TypeF64, sizeof(F64) )
+
+ConsoleGetType( TypeF64 )
+{
+   char* returnBuffer = Con::getReturnBuffer(256);
+   dSprintf(returnBuffer, 256, "%.7f", *((F64 *) dptr) );
+	
+	U32 len = dStrlen(returnBuffer);
+	while (returnBuffer[len-1] == '0')
+		len--;
+
+	if (returnBuffer[len-1] == '.')
+		len--;
+
+	returnBuffer[len] = 0;
+
+   return returnBuffer;
+}
+ConsoleSetType( TypeF64 )
+{
+   if(argc == 1)
+      *((F64 *) dptr) = dAtof64(argv[0]);
+   else
+      Con::printf("(TypeF64) Cannot set multiple args to a single F64.");
+}
+
+//////////////////////////////////////////////////////////////////////////
 // TypeF32Vector
 //////////////////////////////////////////////////////////////////////////
 ConsoleType( floatList, TypeF32Vector, sizeof(Vector<F32>) )
Index: engine/console/consoleTypes.h
===================================================================
--- engine/console/consoleTypes.h	(revision 2053)
+++ engine/console/consoleTypes.h	(working copy)
@@ -20,6 +20,7 @@
 
 // Define Core Console Types
 DefineConsoleType( TypeF32 )
+DefineConsoleType( TypeF64 )
 DefineConsoleType( TypeS8 )
 DefineConsoleType( TypeS32 )
 DefineConsoleType( TypeS32Vector )
Index: engine/console/stringStack.h
===================================================================
--- engine/console/stringStack.h	(revision 2053)
+++ engine/console/stringStack.h	(working copy)
@@ -80,8 +80,18 @@
    void setFloatValue(F64 v)
    {
       validateBufferSize(mStart + 32);
-      dSprintf(mBuffer + mStart, 32, "%g", v);
-      mLen = dStrlen(mBuffer + mStart);
+      dSprintf(mBuffer + mStart, 32, "%.7f", v);
+
+		U32 len = dStrlen(mBuffer + mStart);
+		char *string = mBuffer + mStart;
+		while (string[len-1] == '0')
+			len--;
+
+		if (string[len-1] == '.')
+			len--;
+
+		string[len] = 0;
+      mLen = len;
    }
 
    /// Return a temporary buffer we can use to return data.
Index: engine/platform/platform.h
===================================================================
--- engine/platform/platform.h	(revision 2053)
+++ engine/platform/platform.h	(working copy)
@@ -496,6 +496,7 @@
 
 extern int         dAtoi(const char *str);
 extern float       dAtof(const char *str);
+extern double		 dAtof64(const char *str);
 extern bool        dAtob(const char *str);
 extern unsigned int dAtoui(const char *str);
 extern S64		   dAtoi64(const char *str);
Index: engine/platformWin32/winStrings.cc
===================================================================
--- engine/platformWin32/winStrings.cc	(revision 2053)
+++ engine/platformWin32/winStrings.cc	(working copy)
@@ -247,6 +247,12 @@
    return (F32)atof(str);
 }
 
+F64 dAtof64(const char *str)
+{
+   // Warning: metrowerks crashes when strange strings are passed in '0x [enter]' for example!
+   return atof(str);
+}
+
 bool dAtob(const char *str)
 {
    return !dStricmp(str, "true") || dAtof(str);

Edit: Changed line 32 to F64* not F32*

#5 hey wes, i'm giving this a shot and think i see a potential bug - won't the various sections to truncate trailing zeros truncate "100" to "1" ?

i would propose:

char* truncateFloatString(char* string)
{
   char* p = string + dStrlen(string) - 1;

   while (*p == '0')
   {
      p--;
   }

   if (*p == '.')
   {
      *p = 0;
   }

   return string;
}

#6 hokay, have compiled this in and it seems to work like a charm. nice one wes!

the modifications were a bit different for TGE 1.3.5,
so if anyone's interested in those i'll be happy to post.

imo this should totally go into trunk.

note,
be sure to implement dAtof64 in x86UNIXStrings.cc and macCarbStrings.cc as well!

#7 > be sure to implement dAtof64 in x86UNIXStrings.cc and macCarbStrings.cc as well!
Good call forgot about that.

#8 >won't the various sections to truncate trailing zeros truncate "100" to "1" ?

Yes if there is no decimal point. So far as I have seen the "%.7f" in the sprintf guarantees a decimal value to seven decimal places. If that were to change then code needs to also. The difference with yours is that you will keep zeros in the case of short decimals.

Example: 1.0100000
Original: 1.01
truncateFloatString: 1.0100000

#9 I did find a bug though on line 32 of the second part of the patch should be *((F64 *) dptr) = dAtof64(argv[0]);

#10 A fix for the parser so it supports larger numbers in the script. You have to force it to use a float to get it to support the 64 bit number.

--- engine/console/CMDscan.cc	2009/04/20 19:52:57	2059
+++ engine/console/CMDscan.cc	2009/04/20 20:07:56	2060
@@ -2408,7 +2408,7 @@
 static int Sc_ScanNum()
 {
    CMDtext[CMDleng] = 0;
-   CMDlval.f = dAtof(CMDtext);
+   CMDlval.f = dAtof64(CMDtext);
    return(FLTCONST);
 }

Example:

==>echo(2147483647 + 1);
2147483648
==>echo(2147483648 + 1);
2147483648
==>echo(2147483647.0 + 1);
2147483648
==>echo(2147483648.0 + 1);
2147483649

Irrelevant if all your numbers are in variables.

#11 > truncateFloatString: 1.0100000
doh, i should have seen that.

for those following along, here's the corrected version:

char* truncateFloatString(char* string)
{
   AssertFatal(dStrchr(string, '.') != NULL, "truncateFloatString() - must include a decimal point.");

   char* p = string + dStrlen(string) - 1;

   while (*p == '0')
   {
      p--;
   }

   if (*p == '.')
   {
      p--;
   }

   *(p + 1) = '[[60c1f982386e1]]';

   return string;
}

#12 Very interesting. But just to be sure... These code changes allow the usage of 32 bit signed / unsigned numbers (S32 / U32) in Torque Script without problems, right?

#13 Not exactly it allows you to use numbers much higher than U32, but if you try and use them with functions exposed by the engine that are expecting an S32 you're going to overflow it.

It does allow you to accept F64 values in the engine and those do cover well over the values of a U32 (technically over the values of U64 as well but with less precision.)

If the engine functions you are using expect a string then you can use the full range of the F64 supported in script.

The most important fix is that the script can now support values higher than 10 million or what ever it used to be when it went into scientific notation and those will work with S32. It's only the real big numbers you have to be careful with, but they are now supported.

#14 Thanks a lot for explaining Wes.

#15 Any updates on how to get this to work in T3D?

#16 the mAddS32() et cetera functions should port to T3D no problem.
Wes's fix is much more transparent, but i'm not sure if it'll integrate seamlessly. have you tried ?

#17 If have simply changed the "%g" into "%.9g" to get larger numbers on the following locations:

C:\torque\SDK\engine\console\compiledEval.cc(102): //XXTH2 dSprintf(ret, 32, "%g", arg);
C:\torque\SDK\engine\console\compiler.cc(166): //XXTH2 dSprintf(buf, sizeof(buf), "%g", value);
C:\torque\SDK\engine\console\consoleTypes.cc(187): //XXTH2 dSprintf(returnBuffer, 256, "%g", *((F32 *) dptr) );
C:\torque\SDK\engine\console\stringStack.h(83): //XXTH2 dSprintf(mBuffer + mStart, 32, "%g", v);


There was a post about it, ages ago in TGB Forum.