SH-4310/BUG-2810/MAINT-2794 Better status checking and error logging in Mesh code.

Thu, 27 Jun 2013 13:55:05 -0400

author
Monty Brandenberg <monty@lindenlab.com>
date
Thu, 27 Jun 2013 13:55:05 -0400
changeset 40686
08c1824b4ac3
parent 40685
6335fca454a8
child 40687
ed725f155ee1

SH-4310/BUG-2810/MAINT-2794 Better status checking and error logging in Mesh code.
Pay correct attention to status codes coming back from services. Generate
better and consistent error messages when problems arise. There's more to
do in error handling, need a way to cleanly fail all request types, only
have that for LOD at this point. Do better keeping the HTTP pipeline between
the low and high water marks. This was made challenging because the outer
most code couldn't really see what's going on internally (whose actions are
delayed in a worker thread). More to do here, the debug-like requests don't
honor limits, that will come later. Made retry counts available from llcorehttp
which can be used by the throttle-anticipating logic to advance the count.
It helps but it reinforces the coupling between viewer and server which I
do not like.

indra/llcorehttp/httpresponse.h file | annotate | diff | revisions
indra/newview/llmeshrepository.cpp file | annotate | diff | revisions
indra/newview/llmeshrepository.h file | annotate | diff | revisions
     1.1 --- a/indra/llcorehttp/httpresponse.h	Thu Jun 20 19:18:39 2013 -0400
     1.2 +++ b/indra/llcorehttp/httpresponse.h	Thu Jun 27 13:55:05 2013 -0400
     1.3 @@ -147,8 +147,14 @@
     1.4  	/// Get and set retry attempt information on the request.
     1.5  	void getRetries(unsigned int * retries, unsigned int * retries_503) const
     1.6  		{
     1.7 -			*retries = mRetries;
     1.8 -			*retries_503 = m503Retries;
     1.9 +			if (retries)
    1.10 +			{
    1.11 +				*retries = mRetries;
    1.12 +			}
    1.13 +			if (retries_503)
    1.14 +			{
    1.15 +				*retries_503 = m503Retries;
    1.16 +			}
    1.17  		}
    1.18  
    1.19  	void setRetries(unsigned int retries, unsigned int retries_503)
     2.1 --- a/indra/newview/llmeshrepository.cpp	Thu Jun 20 19:18:39 2013 -0400
     2.2 +++ b/indra/newview/llmeshrepository.cpp	Thu Jun 27 13:55:05 2013 -0400
     2.3 @@ -78,10 +78,12 @@
     2.4  
     2.5  LLMeshRepository gMeshRepo;
     2.6  
     2.7 -const S32 MESH_HEADER_SIZE = 4096;
     2.8 +const S32 MESH_HEADER_SIZE = 4096;						// Important:  assumption is that headers fit in this space
     2.9  const U32 MAX_MESH_REQUESTS_PER_SECOND = 100;
    2.10  const S32 REQUEST_HIGH_WATER_MIN = 32;
    2.11 +const S32 REQUEST_HIGH_WATER_MAX = 80;
    2.12  const S32 REQUEST_LOW_WATER_MIN = 16;
    2.13 +const S32 REQUEST_LOW_WATER_MAX = 40;
    2.14  const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21;		// Size at which requests goes to narrow/slow queue
    2.15  const long LARGE_MESH_XFER_TIMEOUT = 240L;				// Seconds to complete xfer
    2.16  
    2.17 @@ -122,6 +124,7 @@
    2.18  	"medium_lod",
    2.19  	"high_lod"
    2.20  };
    2.21 +const char * const LOG_MESH = "Mesh";
    2.22  
    2.23  // Static data and functions to measure mesh load
    2.24  // time metrics for a new region scene.
    2.25 @@ -211,12 +214,21 @@
    2.26  	}
    2.27  }
    2.28  
    2.29 -S32 LLMeshRepoThread::sActiveHeaderRequests = 0;
    2.30 -S32 LLMeshRepoThread::sActiveLODRequests = 0;
    2.31 +volatile S32 LLMeshRepoThread::sActiveHeaderRequests = 0;
    2.32 +volatile S32 LLMeshRepoThread::sActiveLODRequests = 0;
    2.33  U32	LLMeshRepoThread::sMaxConcurrentRequests = 1;
    2.34  S32 LLMeshRepoThread::sRequestLowWater = REQUEST_LOW_WATER_MIN;
    2.35  S32 LLMeshRepoThread::sRequestHighWater = REQUEST_HIGH_WATER_MIN;
    2.36  
    2.37 +// Base handler class for all mesh users of llcorehttp.
    2.38 +// This is roughly equivalent to a Responder class in
    2.39 +// traditional LL code.  The base is going to perform
    2.40 +// common response/data handling in the inherited
    2.41 +// onCompleted() method.  Derived classes, one for each
    2.42 +// type of HTTP action, define processData() and
    2.43 +// processFailure() methods to customize handling and
    2.44 +// error messages.
    2.45 +//
    2.46  class LLMeshHandlerBase : public LLCore::HttpHandler
    2.47  {
    2.48  public:
    2.49 @@ -225,8 +237,10 @@
    2.50  		  mMeshParams(),
    2.51  		  mProcessed(false),
    2.52  		  mHttpHandle(LLCORE_HTTP_HANDLE_INVALID)
    2.53 -	{}
    2.54 -	virtual ~LLMeshHandlerBase();
    2.55 +		{}
    2.56 +
    2.57 +	virtual ~LLMeshHandlerBase()
    2.58 +		{}
    2.59  
    2.60  protected:
    2.61  	LLMeshHandlerBase(const LLMeshHandlerBase &);				// Not defined
    2.62 @@ -244,6 +258,9 @@
    2.63  };
    2.64  
    2.65  
    2.66 +// Subclass for header fetches.
    2.67 +//
    2.68 +// Thread:  repo
    2.69  class LLMeshHeaderHandler : public LLMeshHandlerBase
    2.70  {
    2.71  public:
    2.72 @@ -265,6 +282,9 @@
    2.73  };
    2.74  
    2.75  
    2.76 +// Subclass for LOD fetches.
    2.77 +//
    2.78 +// Thread:  repo
    2.79  class LLMeshLODHandler : public LLMeshHandlerBase
    2.80  {
    2.81  public:
    2.82 @@ -294,6 +314,9 @@
    2.83  };
    2.84  
    2.85  
    2.86 +// Subclass for skin info fetches.
    2.87 +//
    2.88 +// Thread:  repo
    2.89  class LLMeshSkinInfoHandler : public LLMeshHandlerBase
    2.90  {
    2.91  public:
    2.92 @@ -320,6 +343,9 @@
    2.93  };
    2.94  
    2.95  
    2.96 +// Subclass for decomposition fetches.
    2.97 +//
    2.98 +// Thread:  repo
    2.99  class LLMeshDecompositionHandler : public LLMeshHandlerBase
   2.100  {
   2.101  public:
   2.102 @@ -346,6 +372,9 @@
   2.103  };
   2.104  
   2.105  
   2.106 +// Subclass for physics shape fetches.
   2.107 +//
   2.108 +// Thread:  repo
   2.109  class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase
   2.110  {
   2.111  public:
   2.112 @@ -371,6 +400,7 @@
   2.113  	U32			mOffset;
   2.114  };
   2.115  
   2.116 +
   2.117  void log_upload_error(S32 status, const LLSD& content, std::string stage, std::string model_name)
   2.118  {
   2.119  	// Add notification popup.
   2.120 @@ -555,6 +585,7 @@
   2.121  LLMeshRepoThread::LLMeshRepoThread()
   2.122  : LLThread("mesh repo"),
   2.123    mWaiting(false),
   2.124 +  mHttpRetries(0U),
   2.125    mHttpRequest(NULL),
   2.126    mHttpOptions(NULL),
   2.127    mHttpLargeOptions(NULL),
   2.128 @@ -583,9 +614,9 @@
   2.129  
   2.130  LLMeshRepoThread::~LLMeshRepoThread()
   2.131  {
   2.132 -	LL_INFOS("Mesh") << "Small GETs issued:  "
   2.133 -					 << mHttpGetCount << ", Large GETs issued:  "
   2.134 -					 << mHttpLargeGetCount << LL_ENDL;
   2.135 +	LL_INFOS(LOG_MESH) << "Small GETs issued:  " << mHttpGetCount
   2.136 +					   << ", Large GETs issued:  " << mHttpLargeGetCount
   2.137 +					   << LL_ENDL;
   2.138  
   2.139  	for (http_request_set::iterator iter(mHttpRequestSet.begin());
   2.140  		 iter != mHttpRequestSet.end();
   2.141 @@ -631,10 +662,12 @@
   2.142  	{
   2.143  		if (! mHttpRequestSet.empty())
   2.144  		{
   2.145 +			// Dispatch all HttpHandler notifications
   2.146  			mHttpRequest->update(0L);
   2.147  		}
   2.148  
   2.149  		mWaiting = true;
   2.150 +		ms_sleep(5);
   2.151  		mSignal->wait();
   2.152  		mWaiting = false;
   2.153  		
   2.154 @@ -648,42 +681,49 @@
   2.155  				last_hundred = gFrameTimeSeconds;
   2.156  				count = 0;
   2.157  			}
   2.158 -
   2.159 +			else
   2.160 +			{
   2.161 +				count += mHttpRetries;
   2.162 +			}
   2.163 +			mHttpRetries = 0U;
   2.164 +			
   2.165  			// NOTE: throttling intentionally favors LOD requests over header requests
   2.166  			
   2.167 -			while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND)
   2.168 +			while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && mHttpRequestSet.size() < sRequestHighWater)
   2.169  			{
   2.170 -				if (mMutex)
   2.171 +				if (! mMutex)
   2.172 +				{
   2.173 +					break;
   2.174 +				}
   2.175 +				mMutex->lock();
   2.176 +				LODRequest req = mLODReqQ.front();
   2.177 +				mLODReqQ.pop();
   2.178 +				LLMeshRepository::sLODProcessing--;
   2.179 +				mMutex->unlock();
   2.180 +				if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit
   2.181  				{
   2.182  					mMutex->lock();
   2.183 -					LODRequest req = mLODReqQ.front();
   2.184 -					mLODReqQ.pop();
   2.185 -					LLMeshRepository::sLODProcessing--;
   2.186 +					mLODReqQ.push(req) ; 
   2.187 +					++LLMeshRepository::sLODProcessing;
   2.188  					mMutex->unlock();
   2.189 -					if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit
   2.190 -					{
   2.191 -						mMutex->lock();
   2.192 -						mLODReqQ.push(req) ; 
   2.193 -						++LLMeshRepository::sLODProcessing;
   2.194 -						mMutex->unlock();
   2.195 -					}
   2.196  				}
   2.197  			}
   2.198  
   2.199 -			while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND)
   2.200 +			while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && mHttpRequestSet.size() < sRequestHighWater)
   2.201  			{
   2.202 -				if (mMutex)
   2.203 +				if (! mMutex)
   2.204 +				{
   2.205 +					break;
   2.206 +				}
   2.207 +				mMutex->lock();
   2.208 +				HeaderRequest req = mHeaderReqQ.front();
   2.209 +				mHeaderReqQ.pop();
   2.210 +				mMutex->unlock();
   2.211 +				if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit
   2.212  				{
   2.213  					mMutex->lock();
   2.214 -					HeaderRequest req = mHeaderReqQ.front();
   2.215 -					mHeaderReqQ.pop();
   2.216 +					mHeaderReqQ.push(req) ;
   2.217  					mMutex->unlock();
   2.218 -					if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit
   2.219 -					{
   2.220 -						mMutex->lock();
   2.221 -						mHeaderReqQ.push(req) ;
   2.222 -						mMutex->unlock();
   2.223 -					}
   2.224  				}
   2.225  			}
   2.226  
   2.227 @@ -697,7 +737,7 @@
   2.228  						incomplete.insert(mesh_id);
   2.229  					}
   2.230  				}
   2.231 -				mSkinRequests = incomplete;
   2.232 +				mSkinRequests.swap(incomplete);
   2.233  			}
   2.234  
   2.235  			{ //mDecompositionRequests is protected by mSignal
   2.236 @@ -710,7 +750,7 @@
   2.237  						incomplete.insert(mesh_id);
   2.238  					}
   2.239  				}
   2.240 -				mDecompositionRequests = incomplete;
   2.241 +				mDecompositionRequests.swap(incomplete);
   2.242  			}
   2.243  
   2.244  			{ //mPhysicsShapeRequests is protected by mSignal
   2.245 @@ -723,7 +763,7 @@
   2.246  						incomplete.insert(mesh_id);
   2.247  					}
   2.248  				}
   2.249 -				mPhysicsShapeRequests = incomplete;
   2.250 +				mPhysicsShapeRequests.swap(incomplete);
   2.251  			}
   2.252  		}
   2.253  	}
   2.254 @@ -796,6 +836,10 @@
   2.255  	}
   2.256  }
   2.257  
   2.258 +// Constructs a Cap URL for the mesh.  Prefers a GetMesh2 cap
   2.259 +// over a GetMesh cap and returns what it finds to the caller
   2.260 +// as an int ([1..2]).
   2.261 +//
   2.262  //static 
   2.263  std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id, int * cap_version)
   2.264  {
   2.265 @@ -822,14 +866,26 @@
   2.266  	}
   2.267  	else
   2.268  	{
   2.269 -		llwarns << "Current region does not have GetMesh capability!  Cannot load " << mesh_id << ".mesh" << llendl;
   2.270 +		LL_WARNS_ONCE(LOG_MESH) << "Current region does not have GetMesh capability!  Cannot load "
   2.271 +								<< mesh_id << ".mesh" << LL_ENDL;
   2.272  	}
   2.273  
   2.274  	*cap_version = version;
   2.275  	return http_url;
   2.276  }
   2.277  
   2.278 -// May only be called by repo thread
   2.279 +// Issue an HTTP GET request with byte range using the right
   2.280 +// policy class.  Large requests go to the large request class.
   2.281 +// If the current region supports GetMesh2, we prefer that for
   2.282 +// smaller requests otherwise we try to use the traditional
   2.283 +// GetMesh capability and connection concurrency.
   2.284 +//
   2.285 +// @return		Valid handle or LLCORE_HTTP_HANDLE_INVALID.
   2.286 +//				If the latter, actual status is found in
   2.287 +//				mHttpStatus member which is valid until the
   2.288 +//				next call to this method.
   2.289 +//
   2.290 +// Thread:  repo
   2.291  LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int cap_version,
   2.292  												  size_t offset, size_t len,
   2.293  												  LLCore::HttpHandler * handler)
   2.294 @@ -862,6 +918,11 @@
   2.295  												   handler);
   2.296  		++mHttpLargeGetCount;
   2.297  	}
   2.298 +	if (LLCORE_HTTP_HANDLE_INVALID == handle)
   2.299 +	{
   2.300 +		// Something went wrong, capture the error code for caller.
   2.301 +		mHttpStatus = mHttpRequest->getStatus();
   2.302 +	}
   2.303  	return handle;
   2.304  }
   2.305  
   2.306 @@ -929,12 +990,13 @@
   2.307  			if (!http_url.empty())
   2.308  			{
   2.309  				LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size);
   2.310 -				// LL_WARNS("Mesh") << "MESH:  Issuing Skin Info Request" << LL_ENDL;
   2.311  				LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler);
   2.312  				if (LLCORE_HTTP_HANDLE_INVALID == handle)
   2.313  				{
   2.314 -					// *TODO:  Better error message
   2.315 -					llwarns << "HTTP GET request failed for mesh " << mID << llendl;
   2.316 +					LL_WARNS(LOG_MESH) << "HTTP GET request failed for skin info on mesh " << mID
   2.317 +									   << ".  Reason:  " << mHttpStatus.toString()
   2.318 +									   << " (" << mHttpStatus.toHex() << ")"
   2.319 +									   << LL_ENDL;
   2.320  					delete handler;
   2.321  					ret = false;
   2.322  				}
   2.323 @@ -1019,12 +1081,13 @@
   2.324  			if (!http_url.empty())
   2.325  			{
   2.326  				LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size);
   2.327 -				// LL_WARNS("Mesh") << "MESH:  Issuing Decomp Request" << LL_ENDL;
   2.328  				LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler);
   2.329  				if (LLCORE_HTTP_HANDLE_INVALID == handle)
   2.330  				{
   2.331 -					// *TODO:  Better error message
   2.332 -					llwarns << "HTTP GET request failed for decomposition mesh " << mID << llendl;
   2.333 +					LL_WARNS(LOG_MESH) << "HTTP GET request failed for decomposition mesh " << mID
   2.334 +									   << ".  Reason:  " << mHttpStatus.toString()
   2.335 +									   << " (" << mHttpStatus.toHex() << ")"
   2.336 +									   << LL_ENDL;
   2.337  					delete handler;
   2.338  					ret = false;
   2.339  				}
   2.340 @@ -1108,12 +1171,13 @@
   2.341  			if (!http_url.empty())
   2.342  			{
   2.343  				LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size);
   2.344 -				// LL_WARNS("Mesh") << "MESH:  Issuing Physics Shape Request" << LL_ENDL;
   2.345  				LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler);
   2.346  				if (LLCORE_HTTP_HANDLE_INVALID == handle)
   2.347  				{
   2.348 -					// *TODO:  Better error message
   2.349 -					llwarns << "HTTP GET request failed for physics shape mesh " << mID << llendl;
   2.350 +					LL_WARNS(LOG_MESH) << "HTTP GET request failed for physics shape on mesh " << mID
   2.351 +									   << ".  Reason:  " << mHttpStatus.toString()
   2.352 +									   << " (" << mHttpStatus.toHex() << ")"
   2.353 +									   << LL_ENDL;
   2.354  					delete handler;
   2.355  					ret = false;
   2.356  				}
   2.357 @@ -1192,7 +1256,7 @@
   2.358  	}
   2.359  
   2.360  	//either cache entry doesn't exist or is corrupt, request header from simulator	
   2.361 -	bool retval = true ;
   2.362 +	bool retval = true;
   2.363  	int cap_version(1);
   2.364  	std::string http_url = constructUrl(mesh_params.getSculptID(), &cap_version);
   2.365  	if (!http_url.empty())
   2.366 @@ -1202,12 +1266,13 @@
   2.367  		//NOTE -- this will break of headers ever exceed 4KB		
   2.368  
   2.369  		LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params);
   2.370 -		// LL_WARNS("Mesh") << "MESH:  Issuing Request" << LL_ENDL;
   2.371  		LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler);
   2.372  		if (LLCORE_HTTP_HANDLE_INVALID == handle)
   2.373  		{
   2.374 -			// *TODO:  Better error message
   2.375 -			llwarns << "HTTP GET request failed for mesh " << mID << llendl;
   2.376 +			LL_WARNS(LOG_MESH) << "HTTP GET request failed for mesh header " << mID
   2.377 +							   << ".  Reason:  " << mHttpStatus.toString()
   2.378 +							   << " (" << mHttpStatus.toHex() << ")"
   2.379 +							   << LL_ENDL;
   2.380  			delete handler;
   2.381  			retval = false;
   2.382  		}
   2.383 @@ -1216,8 +1281,8 @@
   2.384  			handler->mHttpHandle = handle;
   2.385  			mHttpRequestSet.insert(handler);
   2.386  			++LLMeshRepository::sHTTPRequestCount;
   2.387 +			++count;
   2.388  		}
   2.389 -		count++;
   2.390  	}
   2.391  
   2.392  	return retval;
   2.393 @@ -1283,12 +1348,13 @@
   2.394  			if (!http_url.empty())
   2.395  			{
   2.396  				LLMeshLODHandler * handler = new LLMeshLODHandler(mesh_params, lod, offset, size);
   2.397 -				// LL_WARNS("Mesh") << "MESH:  Issuing LOD Request" << LL_ENDL;
   2.398  				LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler);
   2.399  				if (LLCORE_HTTP_HANDLE_INVALID == handle)
   2.400  				{
   2.401 -					// *TODO:  Better error message
   2.402 -					llwarns << "HTTP GET request failed for LOD mesh " << mID << llendl;
   2.403 +					LL_WARNS(LOG_MESH) << "HTTP GET request failed for LOD on mesh " << mID
   2.404 +									   << ".  Reason:  " << mHttpStatus.toString()
   2.405 +									   << " (" << mHttpStatus.toHex() << ")"
   2.406 +									   << LL_ENDL;
   2.407  					delete handler;
   2.408  					retval = false;
   2.409  				}
   2.410 @@ -1297,8 +1363,8 @@
   2.411  					handler->mHttpHandle = handle;
   2.412  					mHttpRequestSet.insert(handler);
   2.413  					++LLMeshRepository::sHTTPRequestCount;
   2.414 +					++count;
   2.415  				}
   2.416 -				count++;
   2.417  			}
   2.418  			else
   2.419  			{
   2.420 @@ -1320,6 +1386,7 @@
   2.421  
   2.422  bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size)
   2.423  {
   2.424 +	const LLUUID mesh_id = mesh_params.getSculptID();
   2.425  	LLSD header;
   2.426  	
   2.427  	U32 header_size = 0;
   2.428 @@ -1340,7 +1407,8 @@
   2.429  
   2.430  		if (!LLSDSerialize::fromBinary(header, stream, data_size))
   2.431  		{
   2.432 -			llwarns << "Mesh header parse error.  Not a valid mesh asset!" << llendl;
   2.433 +			LL_WARNS(LOG_MESH) << "Mesh header parse error.  Not a valid mesh asset!  ID:  " << mesh_id
   2.434 +							   << LL_ENDL;
   2.435  			return false;
   2.436  		}
   2.437  
   2.438 @@ -1348,13 +1416,12 @@
   2.439  	}
   2.440  	else
   2.441  	{
   2.442 -		llinfos
   2.443 -			<< "Marking header as non-existent, will not retry." << llendl;
   2.444 +		LL_INFOS(LOG_MESH) << "Non-positive data size.  Marking header as non-existent, will not retry.  ID:  " << mesh_id
   2.445 +						   << LL_ENDL;
   2.446  		header["404"] = 1;
   2.447  	}
   2.448  
   2.449  	{
   2.450 -		LLUUID mesh_id = mesh_params.getSculptID();
   2.451  		
   2.452  		{
   2.453  			LLMutexLock lock(mHeaderMutex);
   2.454 @@ -1416,7 +1483,8 @@
   2.455  
   2.456  		if (!unzip_llsd(skin, stream, data_size))
   2.457  		{
   2.458 -			llwarns << "Mesh skin info parse error.  Not a valid mesh asset!" << llendl;
   2.459 +			LL_WARNS(LOG_MESH) << "Mesh skin info parse error.  Not a valid mesh asset!  ID:  " << mesh_id
   2.460 +							   << LL_ENDL;
   2.461  			return false;
   2.462  		}
   2.463  	}
   2.464 @@ -1444,7 +1512,8 @@
   2.465  
   2.466  		if (!unzip_llsd(decomp, stream, data_size))
   2.467  		{
   2.468 -			llwarns << "Mesh decomposition parse error.  Not a valid mesh asset!" << llendl;
   2.469 +			LL_WARNS(LOG_MESH) << "Mesh decomposition parse error.  Not a valid mesh asset!  ID:  " << mesh_id
   2.470 +							   << LL_ENDL;
   2.471  			return false;
   2.472  		}
   2.473  	}
   2.474 @@ -1860,7 +1929,8 @@
   2.475  
   2.476  	if (mWholeModelUploadURL.empty())
   2.477  	{
   2.478 -		llinfos << "unable to upload, fee request failed" << llendl;
   2.479 +		LL_WARNS(LOG_MESH) << "Missing mesh upload capability, unable to upload, fee request failed."
   2.480 +						   << LL_ENDL;
   2.481  	}
   2.482  	else
   2.483  	{
   2.484 @@ -1941,6 +2011,12 @@
   2.485  		return;
   2.486  	}
   2.487  
   2.488 +	if (!mLoadedQ.empty() || !mUnavailableQ.empty())
   2.489 +	{
   2.490 +		// Ping time-to-load metrics for mesh download operations.
   2.491 +		LLMeshRepository::metricsProgress(0);
   2.492 +	}
   2.493 +	
   2.494  	while (!mLoadedQ.empty())
   2.495  	{
   2.496  		mMutex->lock();
   2.497 @@ -2058,12 +2134,17 @@
   2.498  
   2.499  }
   2.500  
   2.501 -LLMeshHandlerBase::~LLMeshHandlerBase()
   2.502 -{}
   2.503 -
   2.504  void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response)
   2.505  {
   2.506  	mProcessed = true;
   2.507 +
   2.508 +	// Accumulate retries, we'll use these to offset the HTTP
   2.509 +	// count and maybe hold to a throttle better.
   2.510 +	unsigned int retries(0U);
   2.511 +	response->getRetries(NULL, &retries);
   2.512 +	gMeshRepo.mThread->mHttpRetries += retries;
   2.513 +	LLMeshRepository::sHTTPRetryCount += retries;
   2.514 +
   2.515  	LLCore::HttpStatus status(response->getStatus());
   2.516  	if (! status)
   2.517  	{
   2.518 @@ -2090,7 +2171,7 @@
   2.519  			// that support BufferArray directly.
   2.520  			data = new U8[data_size];
   2.521  			body->read(0, (char *) data, data_size);
   2.522 -			LLMeshRepository::sBytesReceived += llmin(data_size, MESH_HEADER_SIZE);
   2.523 +			LLMeshRepository::sBytesReceived += data_size;
   2.524  		}
   2.525  
   2.526  		processData(body, data, data_size);
   2.527 @@ -2100,7 +2181,6 @@
   2.528  
   2.529  	// Release handler
   2.530  	gMeshRepo.mThread->mHttpRequestSet.erase(this);
   2.531 -
   2.532  	delete this;		// Must be last statement
   2.533  }
   2.534  
   2.535 @@ -2112,8 +2192,7 @@
   2.536  		if (! mProcessed)
   2.537  		{
   2.538  			// something went wrong, retry
   2.539 -			llwarns << "Timeout or service unavailable, retrying." << llendl;
   2.540 -			LLMeshRepository::sHTTPRetryCount++;
   2.541 +			LL_WARNS(LOG_MESH) << "Mesh header fetch canceled unexpectedly, retrying." << LL_ENDL;
   2.542  			LLMeshRepoThread::HeaderRequest req(mMeshParams);
   2.543  			LLMutexLock lock(gMeshRepo.mThread->mMutex);
   2.544  			gMeshRepo.mThread->mHeaderReqQ.push(req);
   2.545 @@ -2124,36 +2203,39 @@
   2.546  
   2.547  void LLMeshHeaderHandler::processFailure(LLCore::HttpStatus status)
   2.548  {
   2.549 -	LL_WARNS("Mesh") << "MESH:  Processing Failure" << LL_ENDL;
   2.550  	if (is_retryable(status))
   2.551  	{
   2.552 -		llwarns << "Timeout or service unavailable, retrying." << llendl;
   2.553 -		LLMeshRepository::sHTTPRetryCount++;
   2.554 +		LL_WARNS(LOG_MESH) << "Error during mesh header handling.  Reason:  " << status.toString()
   2.555 +						   << " (" << status.toHex() << ").  Retrying."
   2.556 +						   << LL_ENDL;
   2.557  		LLMeshRepoThread::HeaderRequest req(mMeshParams);
   2.558  		LLMutexLock lock(gMeshRepo.mThread->mMutex);
   2.559  		gMeshRepo.mThread->mHeaderReqQ.push(req);
   2.560  	}
   2.561  	else
   2.562  	{
   2.563 -		// *TODO:  better error message
   2.564 -		llwarns << "Unhandled status." << llendl;
   2.565 +		// *TODO:  Mark mesh unavailable
   2.566 +		LL_WARNS(LOG_MESH) << "Error during mesh header handling.  Reason:  " << status.toString()
   2.567 +						   << " (" << status.toHex() << ").  Not retrying."
   2.568 +						   << LL_ENDL;
   2.569  	}
   2.570  }
   2.571  
   2.572  void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size)
   2.573  {
   2.574 -	// LL_WARNS("Mesh") << "MESH:  Processing Data" << LL_ENDL;
   2.575 +	LLUUID mesh_id = mMeshParams.getSculptID();
   2.576  	bool success = gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size);
   2.577  	llassert(success);
   2.578  	if (! success)
   2.579  	{
   2.580 +		// *TODO:  Mark mesh unavailable
   2.581  		// *TODO:  Get real reason for parse failure here
   2.582 -		llwarns << "Unable to parse mesh header: " << llendl;
   2.583 +		LL_WARNS(LOG_MESH) << "Unable to parse mesh header.  ID:  " << mesh_id
   2.584 +						   << LL_ENDL;
   2.585  	}
   2.586  	else if (data && data_size > 0)
   2.587  	{
   2.588  		// header was successfully retrieved from sim, cache in vfs
   2.589 -		LLUUID mesh_id = mMeshParams.getSculptID();
   2.590  		LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id];
   2.591  
   2.592  		S32 version = header["version"].asInteger();
   2.593 @@ -2192,15 +2274,14 @@
   2.594  			
   2.595  				// zero out the rest of the file 
   2.596  				U8 block[MESH_HEADER_SIZE];
   2.597 -				memset(block, 0, MESH_HEADER_SIZE);
   2.598 -
   2.599 -				while (bytes-file.tell() > MESH_HEADER_SIZE)
   2.600 +				memset(block, 0, sizeof(block));
   2.601 +
   2.602 +				while (bytes-file.tell() > sizeof(block))
   2.603  				{
   2.604 -					file.write(block, MESH_HEADER_SIZE);
   2.605 +					file.write(block, sizeof(block));
   2.606  				}
   2.607  
   2.608  				S32 remaining = bytes-file.tell();
   2.609 -					
   2.610  				if (remaining > 0)
   2.611  				{
   2.612  					file.write(block, remaining);
   2.613 @@ -2216,8 +2297,7 @@
   2.614  	{
   2.615  		if (! mProcessed)
   2.616  		{
   2.617 -			llwarns << "Killed without being processed, retrying." << llendl;
   2.618 -			LLMeshRepository::sHTTPRetryCount++;
   2.619 +			LL_WARNS(LOG_MESH) << "Mesh LOD fetch canceled unexpectedly, retrying." << LL_ENDL;
   2.620  			gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD);
   2.621  		}
   2.622  		LLMeshRepoThread::decActiveLODRequests();
   2.623 @@ -2228,15 +2308,21 @@
   2.624  {
   2.625  	if (is_retryable(status))
   2.626  	{
   2.627 -		llwarns << "Timeout or service unavailable, retrying." << llendl;
   2.628 -		LLMeshRepository::sHTTPRetryCount++;
   2.629 -		// *FIXME:  Is this safe?  Does this need locking?
   2.630 -		gMeshRepo.mThread->loadMeshLOD(mMeshParams, mLOD);
   2.631 +		LL_WARNS(LOG_MESH) << "Error during mesh header handling.  Reason:  " << status.toString()
   2.632 +						   << " (" << status.toHex() << ").  Retrying."
   2.633 +						   << LL_ENDL;
   2.634 +		{
   2.635 +			LLMutexLock lock(gMeshRepo.mThread->mMutex);
   2.636 +
   2.637 +			gMeshRepo.mThread->loadMeshLOD(mMeshParams, mLOD);
   2.638 +		}
   2.639  	}
   2.640  	else
   2.641  	{
   2.642 -		// *TODO:  better error message
   2.643 -		llwarns << "Unhandled status." << llendl;
   2.644 +		// *TODO:  Mark mesh unavailable
   2.645 +		LL_WARNS(LOG_MESH) << "Error during mesh LOD handling.  Reason:  " << status.toString()
   2.646 +						   << " (" << status.toHex() << ").  Not retrying."
   2.647 +						   << LL_ENDL;
   2.648  	}
   2.649  }
   2.650  
   2.651 @@ -2257,6 +2343,7 @@
   2.652  			LLMeshRepository::sCacheBytesWritten += size;
   2.653  		}
   2.654  	}
   2.655 +	// *TODO:  Mark mesh unavailable on error
   2.656  }
   2.657  
   2.658  LLMeshSkinInfoHandler::~LLMeshSkinInfoHandler()
   2.659 @@ -2268,15 +2355,21 @@
   2.660  {
   2.661  	if (is_retryable(status))
   2.662  	{
   2.663 -		llwarns << "Timeout or service unavailable, retrying." << llendl;
   2.664 -		LLMeshRepository::sHTTPRetryCount++;
   2.665 -		// *FIXME:  Is this safe?  Does this need locking?
   2.666 -		gMeshRepo.mThread->loadMeshSkinInfo(mMeshID);
   2.667 +		LL_WARNS(LOG_MESH) << "Error during mesh skin info handling.  Reason:  " << status.toString()
   2.668 +						   << " (" << status.toHex() << ").  Retrying."
   2.669 +						   << LL_ENDL;
   2.670 +		{
   2.671 +			LLMutexLock lock(gMeshRepo.mThread->mMutex);
   2.672 +
   2.673 +			gMeshRepo.mThread->loadMeshSkinInfo(mMeshID);
   2.674 +		}
   2.675  	}
   2.676  	else
   2.677  	{
   2.678 -		// *TODO:  better error message
   2.679 -		llwarns << "Unhandled status." << llendl;
   2.680 +		// *TODO:  Mark mesh unavailable on error
   2.681 +		LL_WARNS(LOG_MESH) << "Error during mesh skin info handling.  Reason:  " << status.toString()
   2.682 +						   << " (" << status.toHex() << ").  Not retrying."
   2.683 +						   << LL_ENDL;
   2.684  	}
   2.685  }
   2.686  
   2.687 @@ -2297,6 +2390,7 @@
   2.688  			file.write(data, size);
   2.689  		}
   2.690  	}
   2.691 +	// *TODO:  Mark mesh unavailable on error
   2.692  }
   2.693  
   2.694  LLMeshDecompositionHandler::~LLMeshDecompositionHandler()
   2.695 @@ -2308,15 +2402,21 @@
   2.696  {
   2.697  	if (is_retryable(status))
   2.698  	{
   2.699 -		llwarns << "Timeout or service unavailable, retrying." << llendl;
   2.700 -		LLMeshRepository::sHTTPRetryCount++;
   2.701 -		// *FIXME:  Is this safe?  Does this need locking?
   2.702 -		gMeshRepo.mThread->loadMeshDecomposition(mMeshID);
   2.703 +		LL_WARNS(LOG_MESH) << "Error during mesh decomposition handling.  Reason:  " << status.toString()
   2.704 +						   << " (" << status.toHex() << ").  Retrying."
   2.705 +						   << LL_ENDL;
   2.706 +		{
   2.707 +			LLMutexLock lock(gMeshRepo.mThread->mMutex);
   2.708 +
   2.709 +			gMeshRepo.mThread->loadMeshDecomposition(mMeshID);
   2.710 +		}
   2.711  	}
   2.712  	else
   2.713  	{
   2.714 -		// *TODO:  better error message
   2.715 -		llwarns << "Unhandled status." << llendl;
   2.716 +		// *TODO:  Mark mesh unavailable on error
   2.717 +		LL_WARNS(LOG_MESH) << "Error during mesh decomposition handling.  Reason:  " << status.toString()
   2.718 +						   << " (" << status.toHex() << ").  Not retrying."
   2.719 +						   << LL_ENDL;
   2.720  	}
   2.721  }
   2.722  
   2.723 @@ -2337,6 +2437,7 @@
   2.724  			file.write(data, size);
   2.725  		}
   2.726  	}
   2.727 +	// *TODO:  Mark mesh unavailable on error
   2.728  }
   2.729  
   2.730  LLMeshPhysicsShapeHandler::~LLMeshPhysicsShapeHandler()
   2.731 @@ -2348,15 +2449,21 @@
   2.732  {
   2.733  	if (is_retryable(status))
   2.734  	{
   2.735 -		llwarns << "Timeout or service unavailable, retrying." << llendl;
   2.736 -		LLMeshRepository::sHTTPRetryCount++;
   2.737 -		// *FIXME:  Is this safe?  Does this need locking?
   2.738 -		gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID);
   2.739 +		LL_WARNS(LOG_MESH) << "Error during mesh physics shape handling.  Reason:  " << status.toString()
   2.740 +						   << " (" << status.toHex() << ").  Retrying."
   2.741 +						   << LL_ENDL;
   2.742 +		{
   2.743 +			LLMutexLock lock(gMeshRepo.mThread->mMutex);
   2.744 +
   2.745 +			gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID);
   2.746 +		}
   2.747  	}
   2.748  	else
   2.749  	{
   2.750 -		// *TODO:  better error message
   2.751 -		llwarns << "Unhandled status." << llendl;
   2.752 +		// *TODO:  Mark mesh unavailable on error
   2.753 +		LL_WARNS(LOG_MESH) << "Error during mesh physics shape handling.  Reason:  " << status.toString()
   2.754 +						   << " (" << status.toHex() << ").  Not retrying."
   2.755 +						   << LL_ENDL;
   2.756  	}
   2.757  }
   2.758  
   2.759 @@ -2377,6 +2484,7 @@
   2.760  			file.write(data, size);
   2.761  		}
   2.762  	}
   2.763 +	// *TODO:  Mark mesh unavailable on error
   2.764  }
   2.765  
   2.766  LLMeshRepository::LLMeshRepository()
   2.767 @@ -2574,12 +2682,15 @@
   2.768  void LLMeshRepository::notifyLoadedMeshes()
   2.769  { //called from main thread
   2.770  	// *FIXME:  Scaling down the setting by a factor of 4 for now to reflect
   2.771 -	// target goal.  May want to rename the setting before release.
   2.772 +	// target goal.  May want to rename the setting before release.  Also
   2.773 +	// want/need to get these in a coordinated fashion from llappcorehttp.
   2.774  	LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests") / 4;
   2.775 -	LLMeshRepoThread::sRequestHighWater = llmax(50 * S32(LLMeshRepoThread::sMaxConcurrentRequests),
   2.776 -												REQUEST_HIGH_WATER_MIN);
   2.777 -	LLMeshRepoThread::sRequestLowWater = llmax(LLMeshRepoThread::sRequestLowWater / 2,
   2.778 -											   REQUEST_LOW_WATER_MIN);
   2.779 +	LLMeshRepoThread::sRequestHighWater = llclamp(10 * S32(LLMeshRepoThread::sMaxConcurrentRequests),
   2.780 +												  REQUEST_HIGH_WATER_MIN,
   2.781 +												  REQUEST_HIGH_WATER_MAX);
   2.782 +	LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2,
   2.783 +												 REQUEST_LOW_WATER_MIN,
   2.784 +												 REQUEST_LOW_WATER_MAX);
   2.785  	
   2.786  	//clean up completed upload threads
   2.787  	for (std::vector<LLMeshUploadThread*>::iterator iter = mUploads.begin(); iter != mUploads.end(); )
   2.788 @@ -2672,6 +2783,10 @@
   2.789  			region_name = gAgent.getRegion()->getName();
   2.790  			mGetMeshCapability = gAgent.getRegion()->getCapability("GetMesh");
   2.791  			mGetMesh2Capability = gAgent.getRegion()->getCapability("GetMesh2");
   2.792 +			LL_DEBUGS(LOG_MESH) << "Retrieving caps for region '" << region_name
   2.793 +								<< "', GetMesh2:  " << mGetMesh2Capability
   2.794 +								<< ", GetMesh:  " << mGetMeshCapability
   2.795 +								<< LL_ENDL;
   2.796  		}
   2.797  	}
   2.798  
   2.799 @@ -2810,9 +2925,6 @@
   2.800  
   2.801  void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVolume* volume)
   2.802  { //called from main thread
   2.803 -	// Manage time-to-load metrics for mesh download operations.
   2.804 -	metricsProgress(0);
   2.805 -	
   2.806  	S32 detail = LLVolumeLODGroup::getVolumeDetailFromScale(volume->getDetail());
   2.807  
   2.808  	//get list of objects waiting to be notified this mesh is loaded
   2.809 @@ -2823,7 +2935,8 @@
   2.810  		//make sure target volume is still valid
   2.811  		if (volume->getNumVolumeFaces() <= 0)
   2.812  		{
   2.813 -			llwarns << "Mesh loading returned empty volume." << llendl;
   2.814 +			LL_WARNS(LOG_MESH) << "Mesh loading returned empty volume.  ID:  " << mesh_params.getSculptID()
   2.815 +							   << LL_ENDL;
   2.816  		}
   2.817  		
   2.818  		{ //update system volume
   2.819 @@ -2836,7 +2949,8 @@
   2.820  			}
   2.821  			else
   2.822  			{
   2.823 -				llwarns << "Couldn't find system volume for given mesh." << llendl;
   2.824 +				LL_WARNS(LOG_MESH) << "Couldn't find system volume for mesh " << mesh_params.getSculptID()
   2.825 +								   << LL_ENDL;
   2.826  			}
   2.827  		}
   2.828  
   2.829 @@ -2856,9 +2970,6 @@
   2.830  
   2.831  void LLMeshRepository::notifyMeshUnavailable(const LLVolumeParams& mesh_params, S32 lod)
   2.832  { //called from main thread
   2.833 -	// Manage time-to-load metrics for mesh download operations.
   2.834 -	metricsProgress(0);
   2.835 -	
   2.836  	//get list of objects waiting to be notified this mesh is loaded
   2.837  	mesh_load_map::iterator obj_iter = mLoadingMeshes[lod].find(mesh_params);
   2.838  
   2.839 @@ -3029,7 +3140,7 @@
   2.840  
   2.841  
   2.842  void LLMeshRepository::uploadModel(std::vector<LLModelInstance>& data, LLVector3& scale, bool upload_textures,
   2.843 -									bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload,
   2.844 +								   bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload,
   2.845  								   LLHandle<LLWholeModelFeeObserver> fee_observer, LLHandle<LLWholeModelUploadObserver> upload_observer)
   2.846  {
   2.847  	LLMeshUploadThread* thread = new LLMeshUploadThread(data, scale, upload_textures, upload_skin, upload_joints, upload_url, 
   2.848 @@ -3058,7 +3169,6 @@
   2.849  	}
   2.850  
   2.851  	return -1;
   2.852 -
   2.853  }
   2.854  
   2.855  void LLMeshUploadThread::decomposeMeshMatrix(LLMatrix4& transformation,
   2.856 @@ -3880,7 +3990,7 @@
   2.857  		metrics["scope"] = "Login";
   2.858  		metrics["start"] = started;
   2.859  		metrics["stop"] = stopped;
   2.860 -		metrics["downloads"] = LLSD::Integer(total_count);
   2.861 +		metrics["fetches"] = LLSD::Integer(total_count);
   2.862  		metrics["teleports"] = LLSD::Integer(metrics_teleport_start_count);
   2.863  		metrics["user_cpu"] = double(user_cpu) / 1.0e6;
   2.864  		metrics["sys_cpu"] = double(sys_cpu) / 1.0e6;
     3.1 --- a/indra/newview/llmeshrepository.h	Thu Jun 20 19:18:39 2013 -0400
     3.2 +++ b/indra/newview/llmeshrepository.h	Thu Jun 27 13:55:05 2013 -0400
     3.3 @@ -221,8 +221,8 @@
     3.4  {
     3.5  public:
     3.6  
     3.7 -	static S32 sActiveHeaderRequests;
     3.8 -	static S32 sActiveLODRequests;
     3.9 +	volatile static S32 sActiveHeaderRequests;
    3.10 +	volatile static S32 sActiveLODRequests;
    3.11  	static U32 sMaxConcurrentRequests;
    3.12  	static S32 sRequestLowWater;
    3.13  	static S32 sRequestHighWater;
    3.14 @@ -323,6 +323,8 @@
    3.15  	pending_lod_map mPendingLOD;
    3.16  
    3.17  	// llcorehttp library interface objects.
    3.18 +	LLCore::HttpStatus					mHttpStatus;
    3.19 +	unsigned int						mHttpRetries;
    3.20  	LLCore::HttpRequest *				mHttpRequest;
    3.21  	LLCore::HttpOptions *				mHttpOptions;
    3.22  	LLCore::HttpOptions *				mHttpLargeOptions;

mercurial