Wed, 19 Jun 2013 13:55:54 -0400
SH-4252 Add second policy class for large mesh asset downloads
Added second mesh class as well as an asset upload class.
Refactored initialization to use less code and more data to
cleanly get http started. Modified mesh to use the new
http class for large requests (>2MB for now). Added additional
timeout setting to llcorehttp to distinguish connection timeout
from transport timeout and are now using transport timeout
values for large asset downloads that may need more time.
1.1 --- a/indra/llcorehttp/_httpinternal.h Fri Jun 07 20:14:52 2013 -0400 1.2 +++ b/indra/llcorehttp/_httpinternal.h Wed Jun 19 13:55:54 2013 -0400 1.3 @@ -98,7 +98,7 @@ 1.4 1.5 // Maxium number of policy classes that can be defined. 1.6 // *TODO: Currently limited to the default class + 1, extend. 1.7 -const int HTTP_POLICY_CLASS_LIMIT = 2; 1.8 +const int HTTP_POLICY_CLASS_LIMIT = 4; 1.9 1.10 // Debug/informational tracing. Used both 1.11 // as a global option and in per-request traces. 1.12 @@ -129,6 +129,7 @@ 1.13 // Retries and time-on-queue are not included and aren't 1.14 // accounted for. 1.15 const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; 1.16 +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L; 1.17 const long HTTP_REQUEST_TIMEOUT_MIN = 0L; 1.18 const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; 1.19
2.1 --- a/indra/llcorehttp/_httpoprequest.cpp Fri Jun 07 20:14:52 2013 -0400 2.2 +++ b/indra/llcorehttp/_httpoprequest.cpp Wed Jun 19 13:55:54 2013 -0400 2.3 @@ -4,7 +4,7 @@ 2.4 * 2.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 2.6 * Second Life Viewer Source Code 2.7 - * Copyright (C) 2012, Linden Research, Inc. 2.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 2.9 * 2.10 * This library is free software; you can redistribute it and/or 2.11 * modify it under the terms of the GNU Lesser General Public 2.12 @@ -110,6 +110,7 @@ 2.13 mReplyFullLength(0), 2.14 mReplyHeaders(NULL), 2.15 mPolicyRetries(0), 2.16 + mPolicy503Retries(0), 2.17 mPolicyRetryAt(HttpTime(0)), 2.18 mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) 2.19 { 2.20 @@ -224,6 +225,7 @@ 2.21 response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); 2.22 } 2.23 response->setContentType(mReplyConType); 2.24 + response->setRetries(mPolicyRetries, mPolicy503Retries); 2.25 2.26 mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); 2.27 2.28 @@ -524,12 +526,19 @@ 2.29 2.30 // Request options 2.31 long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); 2.32 + long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); 2.33 if (mReqOptions) 2.34 - { 2.35 + { 2.36 timeout = mReqOptions->getTimeout(); 2.37 timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); 2.38 + xfer_timeout = mReqOptions->getTransferTimeout(); 2.39 + xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); 2.40 } 2.41 - curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); 2.42 + if (xfer_timeout == 0L) 2.43 + { 2.44 + xfer_timeout = timeout; 2.45 + } 2.46 + curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); 2.47 curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); 2.48 2.49 // Request headers
3.1 --- a/indra/llcorehttp/_httpoprequest.h Fri Jun 07 20:14:52 2013 -0400 3.2 +++ b/indra/llcorehttp/_httpoprequest.h Wed Jun 19 13:55:54 2013 -0400 3.3 @@ -4,7 +4,7 @@ 3.4 * 3.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 3.6 * Second Life Viewer Source Code 3.7 - * Copyright (C) 2012, Linden Research, Inc. 3.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 3.9 * 3.10 * This library is free software; you can redistribute it and/or 3.11 * modify it under the terms of the GNU Lesser General Public 3.12 @@ -187,6 +187,7 @@ 3.13 3.14 // Policy data 3.15 int mPolicyRetries; 3.16 + int mPolicy503Retries; 3.17 HttpTime mPolicyRetryAt; 3.18 int mPolicyRetryLimit; 3.19 }; // end class HttpOpRequest
4.1 --- a/indra/llcorehttp/_httppolicy.cpp Fri Jun 07 20:14:52 2013 -0400 4.2 +++ b/indra/llcorehttp/_httppolicy.cpp Wed Jun 19 13:55:54 2013 -0400 4.3 @@ -140,6 +140,7 @@ 4.4 const int policy_class(op->mReqPolicy); 4.5 4.6 op->mPolicyRetries = 0; 4.7 + op->mPolicy503Retries = 0; 4.8 mState[policy_class].mReadyQueue.push(op); 4.9 } 4.10 4.11 @@ -155,6 +156,7 @@ 4.12 5000000 // ... to every 5.0 S. 4.13 }; 4.14 static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); 4.15 + static const HttpStatus error_503(503); 4.16 4.17 const HttpTime now(totalTime()); 4.18 const int policy_class(op->mReqPolicy); 4.19 @@ -162,6 +164,10 @@ 4.20 const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); 4.21 op->mPolicyRetryAt = now + delta; 4.22 ++op->mPolicyRetries; 4.23 + if (error_503 == op->mStatus) 4.24 + { 4.25 + ++op->mPolicy503Retries; 4.26 + } 4.27 LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) 4.28 << " retry " << op->mPolicyRetries 4.29 << " scheduled for +" << (delta / HttpTime(1000))
5.1 --- a/indra/llcorehttp/httpoptions.cpp Fri Jun 07 20:14:52 2013 -0400 5.2 +++ b/indra/llcorehttp/httpoptions.cpp Wed Jun 19 13:55:54 2013 -0400 5.3 @@ -4,7 +4,7 @@ 5.4 * 5.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 5.6 * Second Life Viewer Source Code 5.7 - * Copyright (C) 2012, Linden Research, Inc. 5.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 5.9 * 5.10 * This library is free software; you can redistribute it and/or 5.11 * modify it under the terms of the GNU Lesser General Public 5.12 @@ -38,6 +38,7 @@ 5.13 mWantHeaders(false), 5.14 mTracing(HTTP_TRACE_OFF), 5.15 mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), 5.16 + mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), 5.17 mRetries(HTTP_RETRY_COUNT_DEFAULT) 5.18 {} 5.19 5.20 @@ -64,6 +65,12 @@ 5.21 } 5.22 5.23 5.24 +void HttpOptions::setTransferTimeout(unsigned int timeout) 5.25 +{ 5.26 + mTransferTimeout = timeout; 5.27 +} 5.28 + 5.29 + 5.30 void HttpOptions::setRetries(unsigned int retries) 5.31 { 5.32 mRetries = retries;
6.1 --- a/indra/llcorehttp/httpoptions.h Fri Jun 07 20:14:52 2013 -0400 6.2 +++ b/indra/llcorehttp/httpoptions.h Wed Jun 19 13:55:54 2013 -0400 6.3 @@ -4,7 +4,7 @@ 6.4 * 6.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 6.6 * Second Life Viewer Source Code 6.7 - * Copyright (C) 2012, Linden Research, Inc. 6.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 6.9 * 6.10 * This library is free software; you can redistribute it and/or 6.11 * modify it under the terms of the GNU Lesser General Public 6.12 @@ -86,6 +86,12 @@ 6.13 return mTimeout; 6.14 } 6.15 6.16 + void setTransferTimeout(unsigned int timeout); 6.17 + unsigned int getTransferTimeout() const 6.18 + { 6.19 + return mTransferTimeout; 6.20 + } 6.21 + 6.22 void setRetries(unsigned int retries); 6.23 unsigned int getRetries() const 6.24 { 6.25 @@ -96,6 +102,7 @@ 6.26 bool mWantHeaders; 6.27 int mTracing; 6.28 unsigned int mTimeout; 6.29 + unsigned int mTransferTimeout; 6.30 unsigned int mRetries; 6.31 6.32 }; // end class HttpOptions
7.1 --- a/indra/llcorehttp/httpresponse.cpp Fri Jun 07 20:14:52 2013 -0400 7.2 +++ b/indra/llcorehttp/httpresponse.cpp Wed Jun 19 13:55:54 2013 -0400 7.3 @@ -4,7 +4,7 @@ 7.4 * 7.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 7.6 * Second Life Viewer Source Code 7.7 - * Copyright (C) 2012, Linden Research, Inc. 7.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 7.9 * 7.10 * This library is free software; you can redistribute it and/or 7.11 * modify it under the terms of the GNU Lesser General Public 7.12 @@ -39,7 +39,9 @@ 7.13 mReplyLength(0U), 7.14 mReplyFullLength(0U), 7.15 mBufferArray(NULL), 7.16 - mHeaders(NULL) 7.17 + mHeaders(NULL), 7.18 + mRetries(0U), 7.19 + m503Retries(0U) 7.20 {} 7.21 7.22
8.1 --- a/indra/llcorehttp/httpresponse.h Fri Jun 07 20:14:52 2013 -0400 8.2 +++ b/indra/llcorehttp/httpresponse.h Wed Jun 19 13:55:54 2013 -0400 8.3 @@ -4,7 +4,7 @@ 8.4 * 8.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 8.6 * Second Life Viewer Source Code 8.7 - * Copyright (C) 2012, Linden Research, Inc. 8.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 8.9 * 8.10 * This library is free software; you can redistribute it and/or 8.11 * modify it under the terms of the GNU Lesser General Public 8.12 @@ -144,6 +144,19 @@ 8.13 mContentType = con_type; 8.14 } 8.15 8.16 + /// Get and set retry attempt information on the request. 8.17 + void getRetries(unsigned int * retries, unsigned int * retries_503) const 8.18 + { 8.19 + *retries = mRetries; 8.20 + *retries_503 = m503Retries; 8.21 + } 8.22 + 8.23 + void setRetries(unsigned int retries, unsigned int retries_503) 8.24 + { 8.25 + mRetries = retries; 8.26 + m503Retries = retries_503; 8.27 + } 8.28 + 8.29 protected: 8.30 // Response data here 8.31 HttpStatus mStatus; 8.32 @@ -153,6 +166,8 @@ 8.33 BufferArray * mBufferArray; 8.34 HttpHeaders * mHeaders; 8.35 std::string mContentType; 8.36 + unsigned int mRetries; 8.37 + unsigned int m503Retries; 8.38 }; 8.39 8.40
9.1 --- a/indra/newview/llappcorehttp.cpp Fri Jun 07 20:14:52 2013 -0400 9.2 +++ b/indra/newview/llappcorehttp.cpp Wed Jun 19 13:55:54 2013 -0400 9.3 @@ -37,11 +37,13 @@ 9.4 : mRequest(NULL), 9.5 mStopHandle(LLCORE_HTTP_HANDLE_INVALID), 9.6 mStopRequested(0.0), 9.7 - mStopped(false), 9.8 - mPolicyDefault(-1), 9.9 - mPolicyTexture(-1), 9.10 - mPolicyMesh(-1) 9.11 -{} 9.12 + mStopped(false) 9.13 +{ 9.14 + for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) 9.15 + { 9.16 + mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; 9.17 + } 9.18 +} 9.19 9.20 9.21 LLAppCoreHttp::~LLAppCoreHttp() 9.22 @@ -53,11 +55,43 @@ 9.23 9.24 void LLAppCoreHttp::init() 9.25 { 9.26 + static const struct 9.27 + { 9.28 + EAppPolicy mPolicy; 9.29 + U32 mDefault; 9.30 + U32 mMin; 9.31 + U32 mMax; 9.32 + U32 mDivisor; 9.33 + std::string mKey; 9.34 + const char * mUsage; 9.35 + } init_data[] = // Default and dynamic values for classes 9.36 + { 9.37 + { 9.38 + AP_TEXTURE, 8, 1, 12, 1, 9.39 + "TextureFetchConcurrency", 9.40 + "texture fetch" 9.41 + }, 9.42 + { 9.43 + AP_MESH, 8, 1, 32, 4, 9.44 + "MeshMaxConcurrentRequests", 9.45 + "mesh fetch" 9.46 + }, 9.47 + { 9.48 + AP_LARGE_MESH, 2, 1, 8, 1, 9.49 + "", 9.50 + "large mesh fetch" 9.51 + }, 9.52 + { 9.53 + AP_UPLOADS, 2, 1, 8, 1, 9.54 + "", 9.55 + "asset upload" 9.56 + } 9.57 + }; 9.58 + 9.59 LLCore::HttpStatus status = LLCore::HttpRequest::createService(); 9.60 if (! status) 9.61 { 9.62 - LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " 9.63 - << status.toString() 9.64 + LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " << status.toString() 9.65 << LL_ENDL; 9.66 } 9.67 9.68 @@ -66,8 +100,7 @@ 9.69 gDirUtilp->getCAFile()); 9.70 if (! status) 9.71 { 9.72 - LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " 9.73 - << status.toString() 9.74 + LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " << status.toString() 9.75 << LL_ENDL; 9.76 } 9.77 9.78 @@ -77,8 +110,7 @@ 9.79 status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); 9.80 if (! status) 9.81 { 9.82 - LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " 9.83 - << status.toString() 9.84 + LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() 9.85 << LL_ENDL; 9.86 } 9.87 9.88 @@ -96,80 +128,72 @@ 9.89 } 9.90 9.91 // Setup default policy and constrain if directed to 9.92 - mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; 9.93 + mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; 9.94 9.95 - // Texture policy will use default for now. 9.96 - mPolicyTexture = mPolicyDefault; 9.97 - static const std::string texture_concur("TextureFetchConcurrency"); 9.98 - if (gSavedSettings.controlExists(texture_concur)) 9.99 + // Setup additional policies based on table and some special rules 9.100 + // *TODO: Make these configurations dynamic later 9.101 + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) 9.102 { 9.103 - U32 concur(llmin(gSavedSettings.getU32(texture_concur), U32(12))); 9.104 + const EAppPolicy policy(init_data[i].mPolicy); 9.105 9.106 - if (concur > 0) 9.107 + // Create a policy class but use default for texture for now. 9.108 + // This also has the side-effect of initializing the default 9.109 + // class to desired values. 9.110 + if (AP_TEXTURE == policy) 9.111 { 9.112 - LLCore::HttpStatus status; 9.113 - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyTexture, 9.114 - LLCore::HttpRequest::CP_CONNECTION_LIMIT, 9.115 - concur); 9.116 - if (! status) 9.117 + mPolicies[policy] = mPolicies[AP_DEFAULT]; 9.118 + } 9.119 + else 9.120 + { 9.121 + mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); 9.122 + if (! mPolicies[policy]) 9.123 { 9.124 - LL_WARNS("Init") << "Unable to set texture fetch concurrency. Reason: " 9.125 - << status.toString() 9.126 + // Use default policy (but don't accidentally modify default) 9.127 + LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage 9.128 + << ". Using default policy." 9.129 << LL_ENDL; 9.130 - } 9.131 - else 9.132 - { 9.133 - LL_INFOS("Init") << "Application settings overriding default texture fetch concurrency. New value: " 9.134 - << concur 9.135 - << LL_ENDL; 9.136 + mPolicies[policy] = mPolicies[AP_DEFAULT]; 9.137 + continue; 9.138 } 9.139 } 9.140 - } 9.141 9.142 - // Create the mesh class 9.143 - mPolicyMesh = LLCore::HttpRequest::createPolicyClass(); 9.144 - if (! mPolicyMesh) 9.145 - { 9.146 - LL_WARNS("Init") << "Failed to create HTTP policy class for Mesh. Using default policy." 9.147 - << LL_ENDL; 9.148 - mPolicyMesh = mPolicyDefault; 9.149 - } 9.150 - else 9.151 - { 9.152 - static const std::string mesh_concur("MeshMaxConcurrentRequests"); 9.153 - if (gSavedSettings.controlExists(mesh_concur)) 9.154 + // Get target connection concurrency value 9.155 + U32 setting(init_data[i].mDefault); 9.156 + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) 9.157 { 9.158 - U32 setting(llmin(gSavedSettings.getU32(mesh_concur), 256U) / 4U); 9.159 - setting = llmax(setting, 2U); 9.160 - 9.161 - if (setting > 0) 9.162 + U32 new_setting(gSavedSettings.getU32(init_data[i].mKey)); 9.163 + if (new_setting) 9.164 { 9.165 - LLCore::HttpStatus status; 9.166 - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyMesh, 9.167 - LLCore::HttpRequest::CP_CONNECTION_LIMIT, 9.168 - setting); 9.169 - if (! status) 9.170 - { 9.171 - LL_WARNS("Init") << "Unable to set mesh fetch concurrency. Reason: " 9.172 - << status.toString() 9.173 - << LL_ENDL; 9.174 - } 9.175 - else 9.176 - { 9.177 - LL_INFOS("Init") << "Application settings overriding default mesh fetch concurrency. New value: " 9.178 - << setting 9.179 - << LL_ENDL; 9.180 - } 9.181 + // Treat zero settings as an ask for default 9.182 + setting = new_setting / init_data[i].mDivisor; 9.183 + setting = llclamp(setting, init_data[i].mMin, init_data[i].mMax); 9.184 } 9.185 } 9.186 + 9.187 + // Set it and report 9.188 + LLCore::HttpStatus status; 9.189 + status = LLCore::HttpRequest::setPolicyClassOption(mPolicies[policy], 9.190 + LLCore::HttpRequest::CP_CONNECTION_LIMIT, 9.191 + setting); 9.192 + if (! status) 9.193 + { 9.194 + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage 9.195 + << " concurrency. Reason: " << status.toString() 9.196 + << LL_ENDL; 9.197 + } 9.198 + else if (setting != init_data[i].mDefault) 9.199 + { 9.200 + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage 9.201 + << " concurrency. New value: " << setting 9.202 + << LL_ENDL; 9.203 + } 9.204 } 9.205 9.206 // Kick the thread 9.207 status = LLCore::HttpRequest::startThread(); 9.208 if (! status) 9.209 { 9.210 - LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " 9.211 - << status.toString() 9.212 + LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " << status.toString() 9.213 << LL_ENDL; 9.214 } 9.215
10.1 --- a/indra/newview/llappcorehttp.h Fri Jun 07 20:14:52 2013 -0400 10.2 +++ b/indra/newview/llappcorehttp.h Wed Jun 19 13:55:54 2013 -0400 10.3 @@ -41,6 +41,19 @@ 10.4 class LLAppCoreHttp : public LLCore::HttpHandler 10.5 { 10.6 public: 10.7 + typedef LLCore::HttpRequest::policy_t policy_t; 10.8 + 10.9 + enum EAppPolicy 10.10 + { 10.11 + AP_DEFAULT, 10.12 + AP_TEXTURE, 10.13 + AP_MESH, 10.14 + AP_LARGE_MESH, 10.15 + AP_UPLOADS, 10.16 + AP_COUNT // Must be last 10.17 + }; 10.18 + 10.19 +public: 10.20 LLAppCoreHttp(); 10.21 ~LLAppCoreHttp(); 10.22 10.23 @@ -65,22 +78,11 @@ 10.24 // Notification when the stop request is complete. 10.25 virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); 10.26 10.27 - // Retrieve the policy class for default operations. 10.28 - int getPolicyDefault() const 10.29 + // Retrieve a policy class identifier for desired 10.30 + // application function. 10.31 + policy_t getPolicy(EAppPolicy policy) const 10.32 { 10.33 - return mPolicyDefault; 10.34 - } 10.35 - 10.36 - // Get the texture fetch policy class. 10.37 - int getPolicyTexture() const 10.38 - { 10.39 - return mPolicyTexture; 10.40 - } 10.41 - 10.42 - // Get the mesh fetch policy class. 10.43 - int getPolicyMesh() const 10.44 - { 10.45 - return mPolicyMesh; 10.46 + return mPolicies[policy]; 10.47 } 10.48 10.49 private: 10.50 @@ -91,9 +93,7 @@ 10.51 LLCore::HttpHandle mStopHandle; 10.52 F64 mStopRequested; 10.53 bool mStopped; 10.54 - int mPolicyDefault; 10.55 - int mPolicyTexture; 10.56 - int mPolicyMesh; 10.57 + policy_t mPolicies[AP_COUNT]; 10.58 }; 10.59 10.60
11.1 --- a/indra/newview/llmeshrepository.cpp Fri Jun 07 20:14:52 2013 -0400 11.2 +++ b/indra/newview/llmeshrepository.cpp Wed Jun 19 13:55:54 2013 -0400 11.3 @@ -80,6 +80,10 @@ 11.4 11.5 const S32 MESH_HEADER_SIZE = 4096; 11.6 const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; 11.7 +const S32 REQUEST_HIGH_WATER_MIN = 32; 11.8 +const S32 REQUEST_LOW_WATER_MIN = 16; 11.9 +const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue 11.10 +const long LARGE_MESH_XFER_TIMEOUT = 240L; // Seconds to complete xfer 11.11 11.12 // Maximum mesh version to support. Three least significant digits are reserved for the minor version, 11.13 // with major version changes indicating a format change that is not backwards compatible and should not 11.14 @@ -210,6 +214,8 @@ 11.15 S32 LLMeshRepoThread::sActiveHeaderRequests = 0; 11.16 S32 LLMeshRepoThread::sActiveLODRequests = 0; 11.17 U32 LLMeshRepoThread::sMaxConcurrentRequests = 1; 11.18 +S32 LLMeshRepoThread::sRequestLowWater = REQUEST_LOW_WATER_MIN; 11.19 +S32 LLMeshRepoThread::sRequestHighWater = REQUEST_HIGH_WATER_MIN; 11.20 11.21 class LLMeshHandlerBase : public LLCore::HttpHandler 11.22 { 11.23 @@ -548,25 +554,37 @@ 11.24 11.25 LLMeshRepoThread::LLMeshRepoThread() 11.26 : LLThread("mesh repo"), 11.27 - mCurlRequest(NULL), 11.28 mWaiting(false), 11.29 mHttpRequest(NULL), 11.30 mHttpOptions(NULL), 11.31 + mHttpLargeOptions(NULL), 11.32 mHttpHeaders(NULL), 11.33 - mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID) 11.34 + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), 11.35 + mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), 11.36 + mHttpPriority(0), 11.37 + mHttpGetCount(0U), 11.38 + mHttpLargeGetCount(0U) 11.39 { 11.40 mMutex = new LLMutex(NULL); 11.41 mHeaderMutex = new LLMutex(NULL); 11.42 mSignal = new LLCondition(NULL); 11.43 mHttpRequest = new LLCore::HttpRequest; 11.44 mHttpOptions = new LLCore::HttpOptions; 11.45 + mHttpLargeOptions = new LLCore::HttpOptions; 11.46 + mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); 11.47 mHttpHeaders = new LLCore::HttpHeaders; 11.48 mHttpHeaders->mHeaders.push_back("Accept: application/vnd.ll.mesh"); 11.49 - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyMesh(); 11.50 + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH); 11.51 + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); 11.52 } 11.53 11.54 + 11.55 LLMeshRepoThread::~LLMeshRepoThread() 11.56 { 11.57 + LL_INFOS("Mesh") << "Small GETs issued: " 11.58 + << mHttpGetCount << ", Large GETs issued: " 11.59 + << mHttpLargeGetCount << LL_ENDL; 11.60 + 11.61 for (http_request_set::iterator iter(mHttpRequestSet.begin()); 11.62 iter != mHttpRequestSet.end(); 11.63 ++iter) 11.64 @@ -584,6 +602,11 @@ 11.65 mHttpOptions->release(); 11.66 mHttpOptions = NULL; 11.67 } 11.68 + if (mHttpLargeOptions) 11.69 + { 11.70 + mHttpLargeOptions->release(); 11.71 + mHttpLargeOptions = NULL; 11.72 + } 11.73 delete mHttpRequest; 11.74 mHttpRequest = NULL; 11.75 delete mMutex; 11.76 @@ -596,7 +619,6 @@ 11.77 11.78 void LLMeshRepoThread::run() 11.79 { 11.80 - mCurlRequest = new LLCurlRequest(); 11.81 LLCDResult res = LLConvexDecomposition::initThread(); 11.82 if (res != LLCD_OK) 11.83 { 11.84 @@ -627,7 +649,7 @@ 11.85 11.86 // NOTE: throttling intentionally favors LOD requests over header requests 11.87 11.88 - while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveLODRequests < sMaxConcurrentRequests) 11.89 + while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND) 11.90 { 11.91 if (mMutex) 11.92 { 11.93 @@ -646,7 +668,7 @@ 11.94 } 11.95 } 11.96 11.97 - while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveHeaderRequests < sMaxConcurrentRequests) 11.98 + while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND) 11.99 { 11.100 if (mMutex) 11.101 { 11.102 @@ -701,8 +723,6 @@ 11.103 } 11.104 mPhysicsShapeRequests = incomplete; 11.105 } 11.106 - 11.107 - mCurlRequest->process(); 11.108 } 11.109 } 11.110 11.111 @@ -716,9 +736,6 @@ 11.112 { 11.113 llwarns << "convex decomposition unable to be quit" << llendl; 11.114 } 11.115 - 11.116 - delete mCurlRequest; 11.117 - mCurlRequest = NULL; 11.118 } 11.119 11.120 void LLMeshRepoThread::loadMeshSkinInfo(const LLUUID& mesh_id) 11.121 @@ -800,6 +817,42 @@ 11.122 return http_url; 11.123 } 11.124 11.125 +// May only be called by repo thread 11.126 +LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, 11.127 + size_t offset, 11.128 + size_t len, 11.129 + LLCore::HttpHandler * handler) 11.130 +{ 11.131 + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); 11.132 + 11.133 + if (len < LARGE_MESH_FETCH_THRESHOLD) 11.134 + { 11.135 + handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, 11.136 + mHttpPriority, 11.137 + url, 11.138 + offset, 11.139 + len, 11.140 + mHttpOptions, 11.141 + mHttpHeaders, 11.142 + handler); 11.143 + ++mHttpGetCount; 11.144 + } 11.145 + else 11.146 + { 11.147 + handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, 11.148 + mHttpPriority, 11.149 + url, 11.150 + offset, 11.151 + len, 11.152 + mHttpLargeOptions, 11.153 + mHttpHeaders, 11.154 + handler); 11.155 + ++mHttpLargeGetCount; 11.156 + } 11.157 + return handle; 11.158 +} 11.159 + 11.160 + 11.161 bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) 11.162 { //protected by mMutex 11.163 11.164 @@ -863,14 +916,7 @@ 11.165 { 11.166 LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size); 11.167 // LL_WARNS("Mesh") << "MESH: Issuing Skin Info Request" << LL_ENDL; 11.168 - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, 11.169 - 0, // *TODO: Get better priority value 11.170 - http_url, 11.171 - offset, 11.172 - size, 11.173 - mHttpOptions, 11.174 - mHttpHeaders, 11.175 - handler); 11.176 + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); 11.177 if (LLCORE_HTTP_HANDLE_INVALID == handle) 11.178 { 11.179 // *TODO: Better error message 11.180 @@ -959,14 +1005,7 @@ 11.181 { 11.182 LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size); 11.183 // LL_WARNS("Mesh") << "MESH: Issuing Decomp Request" << LL_ENDL; 11.184 - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, 11.185 - 0, // *TODO: Get better priority value 11.186 - http_url, 11.187 - offset, 11.188 - size, 11.189 - mHttpOptions, 11.190 - mHttpHeaders, 11.191 - handler); 11.192 + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); 11.193 if (LLCORE_HTTP_HANDLE_INVALID == handle) 11.194 { 11.195 // *TODO: Better error message 11.196 @@ -1054,14 +1093,7 @@ 11.197 { 11.198 LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size); 11.199 // LL_WARNS("Mesh") << "MESH: Issuing Physics Shape Request" << LL_ENDL; 11.200 - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, 11.201 - 0, // *TODO: Get better priority value 11.202 - http_url, 11.203 - offset, 11.204 - size, 11.205 - mHttpOptions, 11.206 - mHttpHeaders, 11.207 - handler); 11.208 + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); 11.209 if (LLCORE_HTTP_HANDLE_INVALID == handle) 11.210 { 11.211 // *TODO: Better error message 11.212 @@ -1154,14 +1186,7 @@ 11.213 11.214 LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); 11.215 // LL_WARNS("Mesh") << "MESH: Issuing Request" << LL_ENDL; 11.216 - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, 11.217 - 0, // *TODO: Get better priority value 11.218 - http_url, 11.219 - 0, 11.220 - MESH_HEADER_SIZE, 11.221 - mHttpOptions, 11.222 - mHttpHeaders, 11.223 - handler); 11.224 + LLCore::HttpHandle handle = getByteRange(http_url, 0, MESH_HEADER_SIZE, handler); 11.225 if (LLCORE_HTTP_HANDLE_INVALID == handle) 11.226 { 11.227 // *TODO: Better error message 11.228 @@ -1241,14 +1266,7 @@ 11.229 { 11.230 LLMeshLODHandler * handler = new LLMeshLODHandler(mesh_params, lod, offset, size); 11.231 // LL_WARNS("Mesh") << "MESH: Issuing LOD Request" << LL_ENDL; 11.232 - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, 11.233 - 0, // *TODO: Get better priority value 11.234 - http_url, 11.235 - offset, 11.236 - size, 11.237 - mHttpOptions, 11.238 - mHttpHeaders, 11.239 - handler); 11.240 + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); 11.241 if (LLCORE_HTTP_HANDLE_INVALID == handle) 11.242 { 11.243 // *TODO: Better error message 11.244 @@ -2537,9 +2555,14 @@ 11.245 11.246 void LLMeshRepository::notifyLoadedMeshes() 11.247 { //called from main thread 11.248 - 11.249 - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); 11.250 - 11.251 + // *FIXME: Scaling down the setting by a factor of 4 for now to reflect 11.252 + // target goal. May want to rename the setting before release. 11.253 + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests") / 4; 11.254 + LLMeshRepoThread::sRequestHighWater = llmax(50 * S32(LLMeshRepoThread::sMaxConcurrentRequests), 11.255 + REQUEST_HIGH_WATER_MIN); 11.256 + LLMeshRepoThread::sRequestLowWater = llmax(LLMeshRepoThread::sRequestLowWater / 2, 11.257 + REQUEST_LOW_WATER_MIN); 11.258 + 11.259 //clean up completed upload threads 11.260 for (std::vector<LLMeshUploadThread*>::iterator iter = mUploads.begin(); iter != mUploads.end(); ) 11.261 { 11.262 @@ -2617,7 +2640,7 @@ 11.263 //call completed callbacks on finished decompositions 11.264 mDecompThread->notifyCompleted(); 11.265 11.266 - if (!mThread->mWaiting) 11.267 + if (!mThread->mWaiting && mPendingRequests.empty()) 11.268 { //curl thread is churning, wait for it to go idle 11.269 return; 11.270 } 11.271 @@ -2644,48 +2667,56 @@ 11.272 mUploadErrorQ.pop(); 11.273 } 11.274 11.275 - S32 push_count = LLMeshRepoThread::sMaxConcurrentRequests-(LLMeshRepoThread::sActiveHeaderRequests+LLMeshRepoThread::sActiveLODRequests); 11.276 - 11.277 - if (push_count > 0) 11.278 + S32 active_count = LLMeshRepoThread::sActiveHeaderRequests + LLMeshRepoThread::sActiveLODRequests; 11.279 + if (active_count < LLMeshRepoThread::sRequestLowWater) 11.280 { 11.281 - //calculate "score" for pending requests 11.282 - 11.283 - //create score map 11.284 - std::map<LLUUID, F32> score_map; 11.285 - 11.286 - for (U32 i = 0; i < 4; ++i) 11.287 + S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; 11.288 + 11.289 + if (mPendingRequests.size() > push_count) 11.290 { 11.291 - for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) 11.292 + // More requests than the high-water limit allows so 11.293 + // sort and forward the most important. 11.294 + 11.295 + //calculate "score" for pending requests 11.296 + 11.297 + //create score map 11.298 + std::map<LLUUID, F32> score_map; 11.299 + 11.300 + for (U32 i = 0; i < 4; ++i) 11.301 { 11.302 - F32 max_score = 0.f; 11.303 - for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) 11.304 + for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) 11.305 { 11.306 - LLViewerObject* object = gObjectList.findObject(*obj_iter); 11.307 - 11.308 - if (object) 11.309 + F32 max_score = 0.f; 11.310 + for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) 11.311 { 11.312 - LLDrawable* drawable = object->mDrawable; 11.313 - if (drawable) 11.314 + LLViewerObject* object = gObjectList.findObject(*obj_iter); 11.315 + 11.316 + if (object) 11.317 { 11.318 - F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); 11.319 - max_score = llmax(max_score, cur_score); 11.320 + LLDrawable* drawable = object->mDrawable; 11.321 + if (drawable) 11.322 + { 11.323 + F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); 11.324 + max_score = llmax(max_score, cur_score); 11.325 + } 11.326 } 11.327 } 11.328 + 11.329 + score_map[iter->first.getSculptID()] = max_score; 11.330 } 11.331 - 11.332 - score_map[iter->first.getSculptID()] = max_score; 11.333 } 11.334 + 11.335 + //set "score" for pending requests 11.336 + for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) 11.337 + { 11.338 + iter->mScore = score_map[iter->mMeshParams.getSculptID()]; 11.339 + } 11.340 + 11.341 + //sort by "score" 11.342 + std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, 11.343 + mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); 11.344 } 11.345 11.346 - //set "score" for pending requests 11.347 - for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) 11.348 - { 11.349 - iter->mScore = score_map[iter->mMeshParams.getSculptID()]; 11.350 - } 11.351 - 11.352 - //sort by "score" 11.353 - std::sort(mPendingRequests.begin(), mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); 11.354 - 11.355 while (!mPendingRequests.empty() && push_count > 0) 11.356 { 11.357 LLMeshRepoThread::LODRequest& request = mPendingRequests.front();
12.1 --- a/indra/newview/llmeshrepository.h Fri Jun 07 20:14:52 2013 -0400 12.2 +++ b/indra/newview/llmeshrepository.h Wed Jun 19 13:55:54 2013 -0400 12.3 @@ -224,13 +224,14 @@ 12.4 static S32 sActiveHeaderRequests; 12.5 static S32 sActiveLODRequests; 12.6 static U32 sMaxConcurrentRequests; 12.7 + static S32 sRequestLowWater; 12.8 + static S32 sRequestHighWater; 12.9 12.10 - LLCurlRequest* mCurlRequest; 12.11 LLMutex* mMutex; 12.12 LLMutex* mHeaderMutex; 12.13 LLCondition* mSignal; 12.14 12.15 - bool mWaiting; 12.16 + volatile bool mWaiting; 12.17 12.18 //map of known mesh headers 12.19 typedef std::map<LLUUID, LLSD> mesh_header_map; 12.20 @@ -324,8 +325,11 @@ 12.21 // llcorehttp library interface objects. 12.22 LLCore::HttpRequest * mHttpRequest; 12.23 LLCore::HttpOptions * mHttpOptions; 12.24 + LLCore::HttpOptions * mHttpLargeOptions; 12.25 LLCore::HttpHeaders * mHttpHeaders; 12.26 LLCore::HttpRequest::policy_t mHttpPolicyClass; 12.27 + LLCore::HttpRequest::policy_t mHttpLargePolicyClass; 12.28 + LLCore::HttpRequest::priority_t mHttpPriority; 12.29 12.30 typedef std::set<LLCore::HttpHandler *> http_request_set; 12.31 http_request_set mHttpRequestSet; // Outstanding HTTP requests 12.32 @@ -373,6 +377,19 @@ 12.33 static void incActiveHeaderRequests(); 12.34 static void decActiveHeaderRequests(); 12.35 12.36 +private: 12.37 + // Issue a GET request to a URL with 'Range' header using 12.38 + // the correct policy class and other attributes. If an invalid 12.39 + // handle is returned, the request failed and caller must retry 12.40 + // or dispose of handler. 12.41 + // 12.42 + // Threads: Repo thread only 12.43 + LLCore::HttpHandle getByteRange(const std::string & url, size_t offset, size_t len, 12.44 + LLCore::HttpHandler * handler); 12.45 + 12.46 +private: 12.47 + U32 mHttpGetCount; 12.48 + U32 mHttpLargeGetCount; 12.49 }; 12.50 12.51 class LLMeshUploadThread : public LLThread
13.1 --- a/indra/newview/lltexturefetch.cpp Fri Jun 07 20:14:52 2013 -0400 13.2 +++ b/indra/newview/lltexturefetch.cpp Wed Jun 19 13:55:54 2013 -0400 13.3 @@ -4,7 +4,7 @@ 13.4 * 13.5 * $LicenseInfo:firstyear=2000&license=viewerlgpl$ 13.6 * Second Life Viewer Source Code 13.7 - * Copyright (C) 2012, Linden Research, Inc. 13.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 13.9 * 13.10 * This library is free software; you can redistribute it and/or 13.11 * modify it under the terms of the GNU Lesser General Public 13.12 @@ -2410,7 +2410,7 @@ 13.13 mHttpHeaders->mHeaders.push_back("Accept: image/x-j2c"); 13.14 mHttpMetricsHeaders = new LLCore::HttpHeaders; 13.15 mHttpMetricsHeaders->mHeaders.push_back("Content-Type: application/llsd+xml"); 13.16 - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyDefault(); 13.17 + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_TEXTURE); 13.18 } 13.19 13.20 LLTextureFetch::~LLTextureFetch()