Game Development Community

dev|Pro Game Development Curriculum

update to bank's libcurl integration

by Thomas -elfprince13- Dickerson · 10/21/2009 (2:46 am) · 14 comments

Make sure you've already installed bank's libcurl integration from here. This resource compiles many of the fixes noted in the comments of that resource, plus adds my own fixes to deal with the Torque Memory Manager and enable use of cookies and form post data from script. I *strongly* recommend the use of Harold "Labrat" Brown's Torquescript urlencode function when working with form data.

[Edit: I should mention that these changes have been tested with TGE 1.5. If you have a different engine version there might be slight differences in how you should go about implementing these. It's probably worth reading through the comment thread for inspiration]

In simCurl.h, find

S32 perform();
		bool isEOF() const;
		void setCancelDownload();

before this add
void setCookieJar(const char *filename);
		void loadCookieFile(const char *filename);
		void addPostField(const char* fieldname, const char* fielddata);


Find,
char url[256];
		char pbuf[64];
		char login[32];
		char password[32];
		char cbProgress[128];
		char cbFinish[128];
		char cbFailed[128];

before this add
bool post;
		struct curl_httppost *formpost;
		struct curl_httppost *lastptr;



Next, in simCurl.cc, find
if(ptr != simcurl->storagedesc.mem.ptr)
		{
			((MemStream*)(s))->~MemStream();
			new(s) MemStream(nsz, ptr);
			s->setPosition(p);
		}

and replace it with
if(ptr != simcurl->storagedesc.mem.ptr)
		{
			((MemStream*)(s))->~MemStream();
			#undef new
			new(s) MemStream(nsz, ptr);
			#if !defined(TORQUE_DISABLE_MEMORY_MANAGER)
			#define new new(__FILE__, __LINE__)
			#endif
			s->setPosition(p);
		}

This will save you blood, sweat, and tears while trying to compile the libcurl integration in MANY environments when Torque Memory Manager is enabled.

Next, ensure that your SimCurl::threadFunction looks like this
void SimCurl::threadFunction(S32 n)
{
	(void)n;
	while(_G.head)
	{
		_G.head->perform();
		Mutex::lockMutex(_G.mutex);
		if (_G.head) 
			_G.head = _G.head->next;
		Mutex::unlockMutex(_G.mutex);
	}
}


Next, find
url[0] = 0;
	pbuf[0] = 0;
	login[0] = 0;
	password[0] = 0;

before it, add
post = false

Find
resumeFrom = 0;
	status = false;
	next = NULL;
After it, add
formpost = NULL;	//added by thomas
	lastptr = NULL;		//added by thomas

Find SimCurl::~SimCurl and replace it with this
SimCurl::~SimCurl()
{
	curl_formfree(formpost);

	
	if(storagedesc.mem.ptr && SIMCURL_STORAGE_BUFFER)
		dFree(storagedesc.mem.ptr);
	
	if (_G.head)  
		_G.head = NULL; 
}

Next, find
void SimCurl::setFileName(const char *filename)

Before it, add

void SimCurl::setCookieJar(const char *filename){
	if(isRunning()) { Con::errorf("Already running:");
		return;
	}
	curl_easy_setopt(curl, CURLOPT_COOKIEJAR, filename);
}

void SimCurl::loadCookieFile(const char *filename){
	if(isRunning()) { Con::errorf("Already running:");
		return;
	}
	curl_easy_setopt(curl, CURLOPT_COOKIEFILE, filename);
}



Find
void SimCurl::setUploadFlag(bool flag)
{
	if(isRunning()) { Con::errorf("Already running:");
		return;
	}
	upload = flag;
}

before it, add
void SimCurl::addPostField(const char *fieldname, const char *fielddata){
	if(isRunning()) { Con::errorf("Already running:");
		return;
	}
	post = true;
	curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME,  fieldname, CURLFORM_COPYCONTENTS, fielddata, CURLFORM_CONTENTSLENGTH, dStrlen(fielddata), CURLFORM_END);
}


find and replace all instances of
ret = CURLE_OBSOLETE4;
with
CURLE_URL_MALFORMAT_USER;

and all instance of
ret = CURLE_OBSOLETE10;
with
ret = CURLE_FTP_USER_PASSWORD_INCORRECT;


Find
if(resumeFrom>0)
         curl_easy_setopt(curl, CURLOPT_RESUME_FROM, resumeFrom);
      curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
      curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, &curl_progress_callback);
      curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);

after it, add
if(post) curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);


If your *SimCurl::readLine begins like this
const U8 *SimCurl::readLine()
{
	U8 *tokPos = storagedesc.mem.c;
	U8 *b = tokPos;

change it to begin like this
const U8 *SimCurl::readLine()
{
	if(!storagedesc.mem.ptr)  
    {  
       static U8 retBuf[1] = "";  
       storagedesc.mem.ptr = NULL;  
       storagedesc.mem.size = 0;  
       storagedesc.mem.c = NULL;  
       storagedesc.mem.rsize = 0;  
       return &retBuf[0];  
    }  
	storagedesc.mem.c[storagedesc.mem.rsize] = 0;
	
	U8 *tokPos = storagedesc.mem.c;
	U8 *b = tokPos;


Finally, find
ConsoleMethod(SimCurl, setUploadFlag, void, 3, 3, "") { object->setUploadFlag( dAtob(argv[2])); }
ConsoleMethod(SimCurl, perform, S32, 2, 2, "") { return object->perform(); }
ConsoleMethod(SimCurl, start, void, 2, 2, "") { object->start(); }

and before it, add this
ConsoleMethod(SimCurl, addPostField, void, 4, 4, "") {  object->addPostField(argv[2], argv[3]);	}
ConsoleMethod(SimCurl, setCookieJar, void, 3, 3, "") {  object->setCookieJar (argv[2]);	}
ConsoleMethod(SimCurl, loadCookieFile, void, 3, 3, "") {  object->loadCookieFile (argv[2]);	}

About the author

C.S. PhD student at Brown University. Project lead for FreeBuild. Administrator, Cemetech tech community. Webmaster for the Village2Village Projects and the Vermont Sustainable Heating Initiative.


#1
10/21/2009 (5:27 am)
Awesome work, Thomas! Many thanks!
#2
10/21/2009 (10:19 am)
Thanks! At some point today I'm going to try and do a write-up of WHY I needed this, which I think a lot of people will find useful and/or interesting.
#3
10/21/2009 (11:48 am)
Thanks for that update!

Quote:At some point today I'm going to try and do a write-up of WHY I needed this, which I think a lot of people will find useful and/or interesting.

Now I'm curious... :)
#4
10/21/2009 (3:15 pm)
thanks for the cookies of chocolate ;D
#5
10/22/2009 (12:12 am)
For those of you who were interested in what I was using this for. Feel free to skip the intro.
#6
10/29/2009 (5:58 pm)
After all your changes, I have an issue in the threadFunction :

Error 1 error C2662: 'SimCurl::perform' : cannot convert 'this' pointer from 'volatile SimCurl' to 'SimCurl &'

I put my old function, and this works perfectly.

What was the goal of this change?
#7
10/29/2009 (9:06 pm)
That was something Konrad figured out in the original discussion thread.

"""
I also found a bug of some sort: it is possible that when (or soon after) deleting the object, perform() gets called. _G.head never evaluates to false even after the deletion, so that was causing me some crashes.

The solution was that I made sure that _G.head is NULL-ed in the destructor, and in threadFunction I call _G.head->next only if _G.head is not null. This has solved all my issues. In code:
...
"""

and changes to the code follow, which are included above.
#8
10/30/2009 (4:42 am)
YEp I read it.

Can you upload you SimCurl.cc somewhere and I will check if I have some discrepancies between my version and yours?

I'm not enough an expert in C++ to solve those things :(
More if you are looking in Konrad comments, it' on the _gead.head.next that he did some changes and here my issue are on the ->Perform function.
SimCurl * pt = (SimCurl *)_G.head;
		if(pt) pt->perform();
and not
_G.head->perform();
#9
10/30/2009 (9:53 am)
http://sfgp.cemetech.net/simCurl.cc

let me know when you're done looking at it.
#10
10/31/2009 (9:37 am)
Thanks, in fact we have some discrepancies.

Major one is the declaration: volatile SimCurl *head, *tail;

in Perform: I use FileStream::createAndOpen instead of in your code ResourceManager->openFileForWrite

All your call to executef are in the "old" format (if I remember weel) with the number of parameter Con::executef(2,...

I'm not an expert to say this or this is better, so if someone with a good background can look at this, it could be useful.

My version is coming from Franck Bignone with your changes. I think, the volatile declaration is coming from tony Richards comment in original thread.

My version:
....
static struct {
	Thread *thread;
	void *mutex;

	volatile SimCurl *head, *tail;
} _G;
....
S32 SimCurl::perform()
{
	Mutex::lockMutex(_G.mutex);
	running = true;
	Mutex::unlockMutex(_G.mutex);
	
	CURLcode ret;
	
	if(storagetype == SIMCURL_STORAGE_FILE)
	{
		if(upload)
		{
			stream = ResourceManager->openStream(storagedesc.filename);
		}
		else
		{
			stream = new FileStream();
			FileStream &s = *(FileStream*)stream;
			if(resumeFrom>0)
			{
				if(!ResourceManager->openFileForWrite(s, storagedesc.filename, File::WriteAppend))
				{
					delete stream;
					stream = NULL;
					ret = CURLE_URL_MALFORMAT_USER;
				}
				else
					s.setPosition(resumeFrom);
			}
			else
				if(!ResourceManager->openFileForWrite(s, storagedesc.filename))
				{
					delete stream;
					stream = NULL;
					ret = CURLE_FTP_USER_PASSWORD_INCORRECT;
				}
		}
	}
	
	if(stream || storagetype == SIMCURL_STORAGE_BUFFER)
	{
		// set all curl options
		if(upload)
		{
			curl_easy_setopt(curl, CURLOPT_READFUNCTION, &read_data);
			curl_easy_setopt(curl, CURLOPT_INFILE, this);
			curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, stream->getStreamSize());
			curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
		} else
		{
			curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
			curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
		}
		
		if(login[0])
		{
			dSprintf(pbuf, sizeof(pbuf), "%s:%s", login, password);
			curl_easy_setopt(curl, CURLOPT_USERPWD, pbuf);
		}
		
		curl_easy_setopt(curl, CURLOPT_URL, url);
		curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
		
		if(resumeFrom>0)
			curl_easy_setopt(curl, CURLOPT_RESUME_FROM, resumeFrom);
		curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
		curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, &curl_progress_callback);
		curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
		
		if(post) curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
		
		ret = curl_easy_perform(curl);
	}
	
	Mutex::lockMutex(_G.mutex);
	running = false;
	Mutex::unlockMutex(_G.mutex);
	
	// if perform() returned an error, transfer is definite failed
	if(ret != CURLE_OK)
		status = ret;//false;
	else
	{
		// for HTTP, perform() doesn't returns error code because it fetches 404 page
		// so we need to check HTTP response code
		long l = -1;
		curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &l);
		status = ret;// (l != 404);	// I guess it have no special meanings in other protocols
	}
	
	// close stream
	if(stream)
	{
		if(upload)
			ResourceManager->closeStream(stream);
		else
			delete stream;
		stream = NULL;
	}
	if(ret==CURLE_OK && cbFinish[0])
		Con::executef(2, cbFinish, this->getIdString());
	if(ret!=CURLE_OK && cbFailed[0])
		Con::executef(2, cbFailed, this->getIdString());
	return ret;// == CURLE_OK;
}
#11
10/31/2009 (1:04 pm)
ah, I'm thinking that you are working with TGEA or T3D, not TGE 1.5?
#12
10/31/2009 (1:21 pm)
T3D exact. This might be the explanation :D
#13
10/31/2009 (1:47 pm)
That would be my guess. In particular the difference between our ConsoleFunction declarations tipped me off.

I've edited the original post to reflect this, and suggest that people not using 1.5 read through this comment thread.
#14
03/02/2010 (12:05 pm)
This resource will help make things a bit more thread-safe: http://www.torquepowered.com/community/resources/view/19409/