Tue, 25 Feb 2014 13:25:40 -0500
merge viewer-release to sunshine-external
1.1 --- a/.hgtags Mon Feb 24 11:33:41 2014 -0500 1.2 +++ b/.hgtags Tue Feb 25 13:25:40 2014 -0500 1.3 @@ -474,3 +474,4 @@ 1.4 0d9b9e50f1a8880e05f15688a9ec7d09e0e81013 3.6.13-release 1.5 5d746de933a98ca17887cde2fece80e9c7ab0b98 3.7.0-release 1.6 dcb4981ce255841b6083d8f65444b65d5a733a17 3.7.1-release 1.7 +b842534cb4d76c9ef87676a62b1d2d19e79c015f 3.7.2-release
2.1 --- a/autobuild.xml Mon Feb 24 11:33:41 2014 -0500 2.2 +++ b/autobuild.xml Tue Feb 25 13:25:40 2014 -0500 2.3 @@ -282,9 +282,9 @@ 2.4 <key>archive</key> 2.5 <map> 2.6 <key>hash</key> 2.7 - <string>aaea644191807f51051cefa2fac11069</string> 2.8 + <string>f7d9b6a9c624364389b71209881f39de</string> 2.9 <key>url</key> 2.10 - <string>http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-darwin-20110316.tar.bz2</string> 2.11 + <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/Darwin/installer/curl-7.24.0-darwin-20130826.tar.bz2</string> 2.12 </map> 2.13 <key>name</key> 2.14 <string>darwin</string> 2.15 @@ -294,9 +294,9 @@ 2.16 <key>archive</key> 2.17 <map> 2.18 <key>hash</key> 2.19 - <string>2d9377951d99a1aa4735cea8d4b5aa71</string> 2.20 + <string>58b7bf45383c1b1bc24afb303b1519c8</string> 2.21 <key>url</key> 2.22 - <string>http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-linux-20110316.tar.bz2</string> 2.23 + <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/Linux/installer/curl-7.24.0-linux-20130826.tar.bz2</string> 2.24 </map> 2.25 <key>name</key> 2.26 <string>linux</string> 2.27 @@ -306,9 +306,9 @@ 2.28 <key>archive</key> 2.29 <map> 2.30 <key>hash</key> 2.31 - <string>fea96aa2a7d513397317194f3d6c979b</string> 2.32 + <string>8d9ccb0277a26bfe3f346c3c49ce4b58</string> 2.33 <key>url</key> 2.34 - <string>http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-windows-20110211.tar.bz2</string> 2.35 + <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/CYGWIN/installer/curl-7.24.0-windows-20130826.tar.bz2</string> 2.36 </map> 2.37 <key>name</key> 2.38 <string>windows</string>
3.1 --- a/indra/edit-me-to-trigger-new-build.txt Mon Feb 24 11:33:41 2014 -0500 3.2 +++ b/indra/edit-me-to-trigger-new-build.txt Tue Feb 25 13:25:40 2014 -0500 3.3 @@ -1,1 +1,1 @@ 3.4 -Mon Apr 15 14:35:39 EDT 2013 3.5 +2014-02-25 10:34
4.1 --- a/indra/llcommon/CMakeLists.txt Mon Feb 24 11:33:41 2014 -0500 4.2 +++ b/indra/llcommon/CMakeLists.txt Tue Feb 25 13:25:40 2014 -0500 4.3 @@ -43,6 +43,7 @@ 4.4 llcriticaldamp.cpp 4.5 llcursortypes.cpp 4.6 lldate.cpp 4.7 + lldeadmantimer.cpp 4.8 lldependencies.cpp 4.9 lldictionary.cpp 4.10 llerror.cpp 4.11 @@ -79,6 +80,7 @@ 4.12 llptrto.cpp 4.13 llprocess.cpp 4.14 llprocessor.cpp 4.15 + llprocinfo.cpp 4.16 llqueuedthread.cpp 4.17 llrand.cpp 4.18 llrefcount.cpp 4.19 @@ -146,6 +148,7 @@ 4.20 lldarray.h 4.21 lldarrayptr.h 4.22 lldate.h 4.23 + lldeadmantimer.h 4.24 lldefs.h 4.25 lldependencies.h 4.26 lldeleteutils.h 4.27 @@ -206,6 +209,7 @@ 4.28 llpriqueuemap.h 4.29 llprocess.h 4.30 llprocessor.h 4.31 + llprocinfo.h 4.32 llptrskiplist.h 4.33 llptrskipmap.h 4.34 llptrto.h 4.35 @@ -323,12 +327,14 @@ 4.36 LL_ADD_INTEGRATION_TEST(bitpack "" "${test_libs}") 4.37 LL_ADD_INTEGRATION_TEST(llbase64 "" "${test_libs}") 4.38 LL_ADD_INTEGRATION_TEST(lldate "" "${test_libs}") 4.39 + LL_ADD_INTEGRATION_TEST(lldeadmantimer "" "${test_libs}") 4.40 LL_ADD_INTEGRATION_TEST(lldependencies "" "${test_libs}") 4.41 LL_ADD_INTEGRATION_TEST(llerror "" "${test_libs}") 4.42 LL_ADD_INTEGRATION_TEST(llframetimer "" "${test_libs}") 4.43 LL_ADD_INTEGRATION_TEST(llinstancetracker "" "${test_libs}") 4.44 LL_ADD_INTEGRATION_TEST(lllazy "" "${test_libs}") 4.45 LL_ADD_INTEGRATION_TEST(llprocessor "" "${test_libs}") 4.46 + LL_ADD_INTEGRATION_TEST(llprocinfo "" "${test_libs}") 4.47 LL_ADD_INTEGRATION_TEST(llrand "" "${test_libs}") 4.48 LL_ADD_INTEGRATION_TEST(llsdserialize "" "${test_libs}") 4.49 LL_ADD_INTEGRATION_TEST(llsingleton "" "${test_libs}")
5.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 5.2 +++ b/indra/llcommon/lldeadmantimer.cpp Tue Feb 25 13:25:40 2014 -0500 5.3 @@ -0,0 +1,188 @@ 5.4 +/** 5.5 +* @file lldeadmantimer.cpp 5.6 +* @brief Simple deadman-switch timer. 5.7 +* @author monty@lindenlab.com 5.8 +* 5.9 +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ 5.10 +* Second Life Viewer Source Code 5.11 +* Copyright (C) 2013, Linden Research, Inc. 5.12 +* 5.13 +* This library is free software; you can redistribute it and/or 5.14 +* modify it under the terms of the GNU Lesser General Public 5.15 +* License as published by the Free Software Foundation; 5.16 +* version 2.1 of the License only. 5.17 +* 5.18 +* This library is distributed in the hope that it will be useful, 5.19 +* but WITHOUT ANY WARRANTY; without even the implied warranty of 5.20 +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 5.21 +* Lesser General Public License for more details. 5.22 +* 5.23 +* You should have received a copy of the GNU Lesser General Public 5.24 +* License along with this library; if not, write to the Free Software 5.25 +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 5.26 +* 5.27 +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA 5.28 +* $/LicenseInfo$ 5.29 +*/ 5.30 + 5.31 + 5.32 +#include "lldeadmantimer.h" 5.33 + 5.34 + 5.35 +// *TODO: Currently, this uses lltimer functions for its time 5.36 +// aspects and this leaks into the apis in the U64s/F64s. Would 5.37 +// like to perhaps switch this over to TSC register-based timers 5.38 +// sometime and drop the overhead some more. 5.39 + 5.40 + 5.41 +// Flag states and their meaning: 5.42 +// mActive mDone Meaning 5.43 +// false false Nothing running, no result available 5.44 +// true false Timer running, no result available 5.45 +// false true Timer finished, result can be read once 5.46 +// true true Not allowed 5.47 +// 5.48 +LLDeadmanTimer::LLDeadmanTimer(F64 horizon, bool inc_cpu) 5.49 + : mHorizon(time_type(llmax(horizon, F64(0.0)) * gClockFrequency)), 5.50 + mActive(false), // If true, a timer is running. 5.51 + mDone(false), // If true, timer has completed and can be read (once) 5.52 + mStarted(U64L(0)), 5.53 + mExpires(U64L(0)), 5.54 + mStopped(U64L(0)), 5.55 + mCount(U64L(0)), 5.56 + mIncCPU(inc_cpu), 5.57 + mUStartCPU(LLProcInfo::time_type(U64L(0))), 5.58 + mUEndCPU(LLProcInfo::time_type(U64L(0))), 5.59 + mSStartCPU(LLProcInfo::time_type(U64L(0))), 5.60 + mSEndCPU(LLProcInfo::time_type(U64L(0))) 5.61 +{} 5.62 + 5.63 + 5.64 +// static 5.65 +LLDeadmanTimer::time_type LLDeadmanTimer::getNow() 5.66 +{ 5.67 + return LLTimer::getCurrentClockCount(); 5.68 +} 5.69 + 5.70 + 5.71 +void LLDeadmanTimer::start(time_type now) 5.72 +{ 5.73 + // *TODO: If active, let's complete an existing timer and save 5.74 + // the result to the side. I think this will be useful later. 5.75 + // For now, wipe out anything in progress, start fresh. 5.76 + 5.77 + if (! now) 5.78 + { 5.79 + now = LLTimer::getCurrentClockCount(); 5.80 + } 5.81 + mActive = true; 5.82 + mDone = false; 5.83 + mStarted = now; 5.84 + mExpires = now + mHorizon; 5.85 + mStopped = now; 5.86 + mCount = U64L(0); 5.87 + if (mIncCPU) 5.88 + { 5.89 + LLProcInfo::getCPUUsage(mUStartCPU, mSStartCPU); 5.90 + } 5.91 +} 5.92 + 5.93 + 5.94 +void LLDeadmanTimer::stop(time_type now) 5.95 +{ 5.96 + if (! mActive) 5.97 + { 5.98 + return; 5.99 + } 5.100 + 5.101 + if (! now) 5.102 + { 5.103 + now = getNow(); 5.104 + } 5.105 + mStopped = now; 5.106 + mActive = false; 5.107 + mDone = true; 5.108 + if (mIncCPU) 5.109 + { 5.110 + LLProcInfo::getCPUUsage(mUEndCPU, mSEndCPU); 5.111 + } 5.112 +} 5.113 + 5.114 + 5.115 +bool LLDeadmanTimer::isExpired(time_type now, F64 & started, F64 & stopped, U64 & count, 5.116 + U64 & user_cpu, U64 & sys_cpu) 5.117 +{ 5.118 + const bool status(isExpired(now, started, stopped, count)); 5.119 + if (status) 5.120 + { 5.121 + user_cpu = U64(mUEndCPU - mUStartCPU); 5.122 + sys_cpu = U64(mSEndCPU - mSStartCPU); 5.123 + } 5.124 + return status; 5.125 +} 5.126 + 5.127 + 5.128 +bool LLDeadmanTimer::isExpired(time_type now, F64 & started, F64 & stopped, U64 & count) 5.129 +{ 5.130 + if (mActive && ! mDone) 5.131 + { 5.132 + if (! now) 5.133 + { 5.134 + now = getNow(); 5.135 + } 5.136 + 5.137 + if (now >= mExpires) 5.138 + { 5.139 + // mStopped from ringBell() is the value we want 5.140 + mActive = false; 5.141 + mDone = true; 5.142 + } 5.143 + } 5.144 + 5.145 + if (! mDone) 5.146 + { 5.147 + return false; 5.148 + } 5.149 + 5.150 + started = mStarted * gClockFrequencyInv; 5.151 + stopped = mStopped * gClockFrequencyInv; 5.152 + count = mCount; 5.153 + mDone = false; 5.154 + 5.155 + return true; 5.156 +} 5.157 + 5.158 + 5.159 +void LLDeadmanTimer::ringBell(time_type now, unsigned int count) 5.160 +{ 5.161 + if (! mActive) 5.162 + { 5.163 + return; 5.164 + } 5.165 + 5.166 + if (! now) 5.167 + { 5.168 + now = getNow(); 5.169 + } 5.170 + 5.171 + if (now >= mExpires) 5.172 + { 5.173 + // Timer has expired, this event will be dropped 5.174 + mActive = false; 5.175 + mDone = true; 5.176 + } 5.177 + else 5.178 + { 5.179 + // Timer renewed, keep going 5.180 + mStopped = now; 5.181 + mExpires = now + mHorizon; 5.182 + mCount += count; 5.183 + if (mIncCPU) 5.184 + { 5.185 + LLProcInfo::getCPUUsage(mUEndCPU, mSEndCPU); 5.186 + } 5.187 + } 5.188 + 5.189 + return; 5.190 +} 5.191 +
6.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 6.2 +++ b/indra/llcommon/lldeadmantimer.h Tue Feb 25 13:25:40 2014 -0500 6.3 @@ -0,0 +1,214 @@ 6.4 +/** 6.5 +* @file lldeadmantimer.h 6.6 +* @brief Interface to a simple event timer with a deadman's switch 6.7 +* @author monty@lindenlab.com 6.8 +* 6.9 +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ 6.10 +* Second Life Viewer Source Code 6.11 +* Copyright (C) 2013, Linden Research, Inc. 6.12 +* 6.13 +* This library is free software; you can redistribute it and/or 6.14 +* modify it under the terms of the GNU Lesser General Public 6.15 +* License as published by the Free Software Foundation; 6.16 +* version 2.1 of the License only. 6.17 +* 6.18 +* This library is distributed in the hope that it will be useful, 6.19 +* but WITHOUT ANY WARRANTY; without even the implied warranty of 6.20 +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 6.21 +* Lesser General Public License for more details. 6.22 +* 6.23 +* You should have received a copy of the GNU Lesser General Public 6.24 +* License along with this library; if not, write to the Free Software 6.25 +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6.26 +* 6.27 +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA 6.28 +* $/LicenseInfo$ 6.29 +*/ 6.30 + 6.31 +#ifndef LL_DEADMANTIMER_H 6.32 +#define LL_DEADMANTIMER_H 6.33 + 6.34 + 6.35 +#include "linden_common.h" 6.36 + 6.37 +#include "lltimer.h" 6.38 +#include "llprocinfo.h" 6.39 + 6.40 + 6.41 +/// @file lldeadmantimer.h 6.42 +/// 6.43 +/// There are interesting user-experienced events in the viewer that 6.44 +/// would seem to have well-defined start and stop points but which 6.45 +/// actually lack such milestones in the code. Such events (like 6.46 +/// time to load meshes after logging in, initial inventory load, 6.47 +/// display name fetch) can be defined somewhat after-the-fact by 6.48 +/// noticing when we no longer perform operations towards their 6.49 +/// completion. This class is intended to help in such applications. 6.50 +/// 6.51 +/// What it implements is a deadman's switch (also known as a 6.52 +/// keepalive switch and a doorbell switch). The basic operation is 6.53 +/// as follows: 6.54 +/// 6.55 +/// * LLDeadmanTimer is instantiated with a horizon value in seconds, 6.56 +/// one for each event of interest. 6.57 +/// * When an event starts, @see start() is invoked to begin a 6.58 +/// timing operation. 6.59 +/// * As operations are performed in service of the event (issuing 6.60 +/// HTTP requests, receiving responses), @see ringBell() is invoked 6.61 +/// to inform the timer that the operation is still active. 6.62 +/// * If the operation is canceled or otherwise terminated, @see 6.63 +/// stop() can be called to end the timing operation. 6.64 +/// * Concurrent with the ringBell() calls, the program makes 6.65 +/// periodic (shorter than the horizon but not too short) calls 6.66 +/// to @see isExpired() to see if the event has expired due to 6.67 +/// either a stop() call or lack of activity (defined as a ringBell() 6.68 +/// call in the previous 'horizon' seconds). If it has expired, 6.69 +/// the caller also receives start, stop and count values for the 6.70 +/// event which the application can then report in whatever manner 6.71 +/// it sees fit. 6.72 +/// * The timer becomes passive after an isExpired() call that returns 6.73 +/// true. It can then be restarted with a new start() call. 6.74 +/// 6.75 +/// Threading: Instances are not thread-safe. They also use 6.76 +/// timing code from lltimer.h which is also unsafe. 6.77 +/// 6.78 +/// Allocation: Not refcounted, may be stack or heap allocated. 6.79 +/// 6.80 + 6.81 +class LL_COMMON_API LLDeadmanTimer 6.82 +{ 6.83 +public: 6.84 + /// Public types 6.85 + 6.86 + /// Low-level time type chosen for compatibility with 6.87 + /// LLTimer::getCurrentClockCount() which is the basis 6.88 + /// of time operations in this class. This is likely 6.89 + /// to change in a future version in a move to TSC-based 6.90 + /// timing. 6.91 + typedef U64 time_type; 6.92 + 6.93 +public: 6.94 + /// Construct and initialize an LLDeadmanTimer 6.95 + /// 6.96 + /// @param horizon Time, in seconds, after the last @see ringBell() 6.97 + /// call at which point the timer will consider itself 6.98 + /// expired. 6.99 + /// 6.100 + /// @param inc_cpu If true, gather system and user cpu stats while 6.101 + /// running the timer. This does require more syscalls 6.102 + /// during updates. If false, cpu usage data isn't 6.103 + /// collected and will be zero if queried. 6.104 + LLDeadmanTimer(F64 horizon, bool inc_cpu); 6.105 + 6.106 + ~LLDeadmanTimer() 6.107 + {} 6.108 + 6.109 +private: 6.110 + LLDeadmanTimer(const LLDeadmanTimer &); // Not defined 6.111 + void operator=(const LLDeadmanTimer &); // Not defined 6.112 + 6.113 +public: 6.114 + /// Get the current time. Zero-basis for this time 6.115 + /// representation is not defined and is different on 6.116 + /// different platforms. Do not attempt to compute 6.117 + /// negative times relative to the first value returned, 6.118 + /// there may not be enough 'front porch' on the range 6.119 + /// to prevent wraparound. 6.120 + /// 6.121 + /// Note: Implementation is expected to change in a 6.122 + /// future release as well. 6.123 + /// 6.124 + static time_type getNow(); 6.125 + 6.126 + /// Begin timing. If the timer is already active, it is reset 6.127 + /// and timing begins now. 6.128 + /// 6.129 + /// @param now Current time as returned by @see 6.130 + /// LLTimer::getCurrentClockCount(). If zero, 6.131 + /// method will lookup current time. 6.132 + /// 6.133 + void start(time_type now); 6.134 + 6.135 + /// End timing. Actively declare the end of the event independent 6.136 + /// of the deadman's switch operation. @see isExpired() will return 6.137 + /// true and appropriate values will be returned. 6.138 + /// 6.139 + /// @param now Current time as returned by @see 6.140 + /// LLTimer::getCurrentClockCount(). If zero, 6.141 + /// method will lookup current time. 6.142 + /// 6.143 + void stop(time_type now); 6.144 + 6.145 + /// Declare that something interesting happened. This has two 6.146 + /// effects on an unexpired-timer. 1) The expiration time 6.147 + /// is extended for 'horizon' seconds after the 'now' value. 6.148 + /// 2) An internal counter associated with the event is incremented 6.149 + /// by the @ref count parameter. This count is returned via the 6.150 + /// @see isExpired() method. 6.151 + /// 6.152 + /// @param now Current time as returned by @see 6.153 + /// LLTimer::getCurrentClockCount(). If zero, 6.154 + /// method will lookup current time. 6.155 + /// 6.156 + /// @param count Count of events to be associated with 6.157 + /// this bell ringing. 6.158 + /// 6.159 + void ringBell(time_type now, unsigned int count); 6.160 + 6.161 + /// Checks the status of the timer. If the timer has expired, 6.162 + /// also returns various timer-related stats. Unlike ringBell(), 6.163 + /// does not extend the horizon, it only checks for expiration. 6.164 + /// 6.165 + /// @param now Current time as returned by @see 6.166 + /// LLTimer::getCurrentClockCount(). If zero, 6.167 + /// method will lookup current time. 6.168 + /// 6.169 + /// @param started If expired, the starting time of the event is 6.170 + /// returned to the caller via this reference. 6.171 + /// 6.172 + /// @param stopped If expired, the ending time of the event is 6.173 + /// returned to the caller via this reference. 6.174 + /// Ending time will be that provided in the 6.175 + /// stop() method or the last ringBell() call 6.176 + /// leading to expiration, whichever (stop() call 6.177 + /// or notice of expiration) happened first. 6.178 + /// 6.179 + /// @param count If expired, the number of ringBell() calls 6.180 + /// made prior to expiration. 6.181 + /// 6.182 + /// @param user_cpu Amount of CPU spent in user mode by the process 6.183 + /// during the event. Value in microseconds and will 6.184 + /// read zero if not enabled by the constructor. 6.185 + /// 6.186 + /// @param sys_cpu Amount of CPU spent in system mode by the process. 6.187 + /// 6.188 + /// @return true if the timer has expired, false otherwise. 6.189 + /// If true, it also returns the started, 6.190 + /// stopped and count values otherwise these are 6.191 + /// left unchanged. 6.192 + /// 6.193 + bool isExpired(time_type now, F64 & started, F64 & stopped, U64 & count, 6.194 + U64 & user_cpu, U64 & sys_cpu); 6.195 + 6.196 + /// Identical to the six-arugment form except it does without the 6.197 + /// CPU time return if the caller isn't interested in it. 6.198 + bool isExpired(time_type now, F64 & started, F64 & stopped, U64 & count); 6.199 + 6.200 +protected: 6.201 + time_type mHorizon; 6.202 + bool mActive; 6.203 + bool mDone; 6.204 + time_type mStarted; 6.205 + time_type mExpires; 6.206 + time_type mStopped; 6.207 + time_type mCount; 6.208 + 6.209 + const bool mIncCPU; // Include CPU metrics in timer 6.210 + LLProcInfo::time_type mUStartCPU; 6.211 + LLProcInfo::time_type mUEndCPU; 6.212 + LLProcInfo::time_type mSStartCPU; 6.213 + LLProcInfo::time_type mSEndCPU; 6.214 +}; 6.215 + 6.216 + 6.217 +#endif // LL_DEADMANTIMER_H
7.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 7.2 +++ b/indra/llcommon/llprocinfo.cpp Tue Feb 25 13:25:40 2014 -0500 7.3 @@ -0,0 +1,94 @@ 7.4 +/** 7.5 +* @file llprocinfo.cpp 7.6 +* @brief Process, cpu and resource usage information APIs. 7.7 +* @author monty@lindenlab.com 7.8 +* 7.9 +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ 7.10 +* Second Life Viewer Source Code 7.11 +* Copyright (C) 2013, Linden Research, Inc. 7.12 +* 7.13 +* This library is free software; you can redistribute it and/or 7.14 +* modify it under the terms of the GNU Lesser General Public 7.15 +* License as published by the Free Software Foundation; 7.16 +* version 2.1 of the License only. 7.17 +* 7.18 +* This library is distributed in the hope that it will be useful, 7.19 +* but WITHOUT ANY WARRANTY; without even the implied warranty of 7.20 +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 7.21 +* Lesser General Public License for more details. 7.22 +* 7.23 +* You should have received a copy of the GNU Lesser General Public 7.24 +* License along with this library; if not, write to the Free Software 7.25 +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 7.26 +* 7.27 +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA 7.28 +* $/LicenseInfo$ 7.29 +*/ 7.30 + 7.31 + 7.32 +#include "llprocinfo.h" 7.33 + 7.34 +#if LL_WINDOWS 7.35 + 7.36 +#define PSAPI_VERSION 1 7.37 +#include "windows.h" 7.38 +#include "psapi.h" 7.39 + 7.40 +#elif LL_DARWIN 7.41 + 7.42 +#include <sys/resource.h> 7.43 +#include <mach/mach.h> 7.44 + 7.45 +#else 7.46 + 7.47 +#include <sys/time.h> 7.48 +#include <sys/resource.h> 7.49 + 7.50 +#endif // LL_WINDOWS/LL_DARWIN 7.51 + 7.52 + 7.53 +// static 7.54 +void LLProcInfo::getCPUUsage(time_type & user_time, time_type & system_time) 7.55 +{ 7.56 +#if LL_WINDOWS 7.57 + 7.58 + HANDLE self(GetCurrentProcess()); // Does not have to be closed 7.59 + FILETIME ft_dummy, ft_system, ft_user; 7.60 + 7.61 + GetProcessTimes(self, &ft_dummy, &ft_dummy, &ft_system, &ft_user); 7.62 + ULARGE_INTEGER uli; 7.63 + uli.u.LowPart = ft_system.dwLowDateTime; 7.64 + uli.u.HighPart = ft_system.dwHighDateTime; 7.65 + system_time = uli.QuadPart / U64L(10); // Convert to uS 7.66 + uli.u.LowPart = ft_user.dwLowDateTime; 7.67 + uli.u.HighPart = ft_user.dwHighDateTime; 7.68 + user_time = uli.QuadPart / U64L(10); 7.69 + 7.70 +#elif LL_DARWIN 7.71 + 7.72 + struct rusage usage; 7.73 + 7.74 + if (getrusage(RUSAGE_SELF, &usage)) 7.75 + { 7.76 + user_time = system_time = time_type(0U); 7.77 + return; 7.78 + } 7.79 + user_time = U64(usage.ru_utime.tv_sec) * U64L(1000000) + usage.ru_utime.tv_usec; 7.80 + system_time = U64(usage.ru_stime.tv_sec) * U64L(1000000) + usage.ru_stime.tv_usec; 7.81 + 7.82 +#else // Linux 7.83 + 7.84 + struct rusage usage; 7.85 + 7.86 + if (getrusage(RUSAGE_SELF, &usage)) 7.87 + { 7.88 + user_time = system_time = time_type(0U); 7.89 + return; 7.90 + } 7.91 + user_time = U64(usage.ru_utime.tv_sec) * U64L(1000000) + usage.ru_utime.tv_usec; 7.92 + system_time = U64(usage.ru_stime.tv_sec) * U64L(1000000) + usage.ru_stime.tv_usec; 7.93 + 7.94 +#endif // LL_WINDOWS/LL_DARWIN/Linux 7.95 +} 7.96 + 7.97 +
8.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 8.2 +++ b/indra/llcommon/llprocinfo.h Tue Feb 25 13:25:40 2014 -0500 8.3 @@ -0,0 +1,68 @@ 8.4 +/** 8.5 +* @file llprocinfo.h 8.6 +* @brief Interface to process/cpu/resource information services. 8.7 +* @author monty@lindenlab.com 8.8 +* 8.9 +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ 8.10 +* Second Life Viewer Source Code 8.11 +* Copyright (C) 2013, Linden Research, Inc. 8.12 +* 8.13 +* This library is free software; you can redistribute it and/or 8.14 +* modify it under the terms of the GNU Lesser General Public 8.15 +* License as published by the Free Software Foundation; 8.16 +* version 2.1 of the License only. 8.17 +* 8.18 +* This library is distributed in the hope that it will be useful, 8.19 +* but WITHOUT ANY WARRANTY; without even the implied warranty of 8.20 +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 8.21 +* Lesser General Public License for more details. 8.22 +* 8.23 +* You should have received a copy of the GNU Lesser General Public 8.24 +* License along with this library; if not, write to the Free Software 8.25 +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 8.26 +* 8.27 +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA 8.28 +* $/LicenseInfo$ 8.29 +*/ 8.30 + 8.31 +#ifndef LL_PROCINFO_H 8.32 +#define LL_PROCINFO_H 8.33 + 8.34 + 8.35 +#include "linden_common.h" 8.36 + 8.37 +/// @file llprocinfo.h 8.38 +/// 8.39 +/// Right now, this is really a namespace disguised as a class. 8.40 +/// It wraps some types and functions to return information about 8.41 +/// process resource consumption in a non-OS-specific manner. 8.42 +/// 8.43 +/// Threading: No instances so that's thread-safe. Implementations 8.44 +/// of static functions should be thread-safe, they mostly involve 8.45 +/// direct syscall invocations. 8.46 +/// 8.47 +/// Allocation: Not instantiatable. 8.48 + 8.49 +class LL_COMMON_API LLProcInfo 8.50 +{ 8.51 +public: 8.52 + /// Public types 8.53 + 8.54 + typedef U64 time_type; /// Relative microseconds 8.55 + 8.56 +private: 8.57 + LLProcInfo(); // Not defined 8.58 + ~LLProcInfo(); // Not defined 8.59 + LLProcInfo(const LLProcInfo &); // Not defined 8.60 + void operator=(const LLProcInfo &); // Not defined 8.61 + 8.62 +public: 8.63 + /// Get accumulated system and user CPU time in 8.64 + /// microseconds. Syscalls involved in every invocation. 8.65 + /// 8.66 + /// Threading: expected to be safe. 8.67 + static void getCPUUsage(time_type & user_time, time_type & system_time); 8.68 +}; 8.69 + 8.70 + 8.71 +#endif // LL_PROCINFO_H
9.1 --- a/indra/llcommon/llthread.cpp Mon Feb 24 11:33:41 2014 -0500 9.2 +++ b/indra/llcommon/llthread.cpp Tue Feb 25 13:25:40 2014 -0500 9.3 @@ -3,7 +3,7 @@ 9.4 * 9.5 * $LicenseInfo:firstyear=2004&license=viewerlgpl$ 9.6 * Second Life Viewer Source Code 9.7 - * Copyright (C) 2010, Linden Research, Inc. 9.8 + * Copyright (C) 2010-2013, Linden Research, Inc. 9.9 * 9.10 * This library is free software; you can redistribute it and/or 9.11 * modify it under the terms of the GNU Lesser General Public 9.12 @@ -373,6 +373,36 @@ 9.13 #endif 9.14 } 9.15 9.16 +bool LLMutex::trylock() 9.17 +{ 9.18 + if(isSelfLocked()) 9.19 + { //redundant lock 9.20 + mCount++; 9.21 + return true; 9.22 + } 9.23 + 9.24 + apr_status_t status(apr_thread_mutex_trylock(mAPRMutexp)); 9.25 + if (APR_STATUS_IS_EBUSY(status)) 9.26 + { 9.27 + return false; 9.28 + } 9.29 + 9.30 +#if MUTEX_DEBUG 9.31 + // Have to have the lock before we can access the debug info 9.32 + U32 id = LLThread::currentID(); 9.33 + if (mIsLocked[id] != FALSE) 9.34 + llerrs << "Already locked in Thread: " << id << llendl; 9.35 + mIsLocked[id] = TRUE; 9.36 +#endif 9.37 + 9.38 +#if LL_DARWIN 9.39 + mLockingThread = LLThread::currentID(); 9.40 +#else 9.41 + mLockingThread = sThreadID; 9.42 +#endif 9.43 + return true; 9.44 +} 9.45 + 9.46 void LLMutex::unlock() 9.47 { 9.48 if (mCount > 0)
10.1 --- a/indra/llcommon/llthread.h Mon Feb 24 11:33:41 2014 -0500 10.2 +++ b/indra/llcommon/llthread.h Tue Feb 25 13:25:40 2014 -0500 10.3 @@ -4,7 +4,7 @@ 10.4 * 10.5 * $LicenseInfo:firstyear=2004&license=viewerlgpl$ 10.6 * Second Life Viewer Source Code 10.7 - * Copyright (C) 2010, Linden Research, Inc. 10.8 + * Copyright (C) 2010-2013, Linden Research, Inc. 10.9 * 10.10 * This library is free software; you can redistribute it and/or 10.11 * modify it under the terms of the GNU Lesser General Public 10.12 @@ -156,7 +156,8 @@ 10.13 virtual ~LLMutex(); 10.14 10.15 void lock(); // blocks 10.16 - void unlock(); 10.17 + bool trylock(); // non-blocking, returns true if lock held. 10.18 + void unlock(); // undefined behavior when called on mutex not being held 10.19 bool isLocked(); // non-blocking, but does do a lock/unlock so not free 10.20 bool isSelfLocked(); //return true if locked in a same thread 10.21 U32 lockingThread() const; //get ID of locking thread 10.22 @@ -174,6 +175,8 @@ 10.23 #endif 10.24 }; 10.25 10.26 +//============================================================================ 10.27 + 10.28 // Actually a condition/mutex pair (since each condition needs to be associated with a mutex). 10.29 class LL_COMMON_API LLCondition : public LLMutex 10.30 { 10.31 @@ -189,6 +192,8 @@ 10.32 apr_thread_cond_t *mAPRCondp; 10.33 }; 10.34 10.35 +//============================================================================ 10.36 + 10.37 class LLMutexLock 10.38 { 10.39 public: 10.40 @@ -210,6 +215,43 @@ 10.41 10.42 //============================================================================ 10.43 10.44 +// Scoped locking class similar in function to LLMutexLock but uses 10.45 +// the trylock() method to conditionally acquire lock without 10.46 +// blocking. Caller resolves the resulting condition by calling 10.47 +// the isLocked() method and either punts or continues as indicated. 10.48 +// 10.49 +// Mostly of interest to callers needing to avoid stalls who can 10.50 +// guarantee another attempt at a later time. 10.51 + 10.52 +class LLMutexTrylock 10.53 +{ 10.54 +public: 10.55 + LLMutexTrylock(LLMutex* mutex) 10.56 + : mMutex(mutex), 10.57 + mLocked(false) 10.58 + { 10.59 + if (mMutex) 10.60 + mLocked = mMutex->trylock(); 10.61 + } 10.62 + 10.63 + ~LLMutexTrylock() 10.64 + { 10.65 + if (mMutex && mLocked) 10.66 + mMutex->unlock(); 10.67 + } 10.68 + 10.69 + bool isLocked() const 10.70 + { 10.71 + return mLocked; 10.72 + } 10.73 + 10.74 +private: 10.75 + LLMutex* mMutex; 10.76 + bool mLocked; 10.77 +}; 10.78 + 10.79 +//============================================================================ 10.80 + 10.81 void LLThread::lockData() 10.82 { 10.83 mDataLock->lock();
11.1 --- a/indra/llcommon/lltimer.h Mon Feb 24 11:33:41 2014 -0500 11.2 +++ b/indra/llcommon/lltimer.h Tue Feb 25 13:25:40 2014 -0500 11.3 @@ -146,6 +146,13 @@ 11.4 } 11.5 } 11.6 11.7 +// These are really statics but they've been global for awhile 11.8 +// and they're material to other timing classes. If you are 11.9 +// not implementing a timer class, do not use these directly. 11.10 +extern LL_COMMON_API F64 gClockFrequency; 11.11 +extern LL_COMMON_API F64 gClockFrequencyInv; 11.12 +extern LL_COMMON_API F64 gClocksToMicroseconds; 11.13 + 11.14 // Correction factor used by time_corrected() above. 11.15 extern LL_COMMON_API S32 gUTCOffset; 11.16
12.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 12.2 +++ b/indra/llcommon/tests/lldeadmantimer_test.cpp Tue Feb 25 13:25:40 2014 -0500 12.3 @@ -0,0 +1,628 @@ 12.4 +/** 12.5 + * @file lldeadmantimer_test.cpp 12.6 + * @brief Tests for the LLDeadmanTimer class. 12.7 + * 12.8 + * $LicenseInfo:firstyear=2013&license=viewerlgpl$ 12.9 + * Second Life Viewer Source Code 12.10 + * Copyright (C) 2013, Linden Research, Inc. 12.11 + * 12.12 + * This library is free software; you can redistribute it and/or 12.13 + * modify it under the terms of the GNU Lesser General Public 12.14 + * License as published by the Free Software Foundation; 12.15 + * version 2.1 of the License only. 12.16 + * 12.17 + * This library is distributed in the hope that it will be useful, 12.18 + * but WITHOUT ANY WARRANTY; without even the implied warranty of 12.19 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12.20 + * Lesser General Public License for more details. 12.21 + * 12.22 + * You should have received a copy of the GNU Lesser General Public 12.23 + * License along with this library; if not, write to the Free Software 12.24 + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 12.25 + * 12.26 + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA 12.27 + * $/LicenseInfo$ 12.28 + */ 12.29 + 12.30 +#include "linden_common.h" 12.31 + 12.32 +#include "../lldeadmantimer.h" 12.33 +#include "../llsd.h" 12.34 +#include "../lltimer.h" 12.35 + 12.36 +#include "../test/lltut.h" 12.37 + 12.38 +// Convert between floating point time deltas and U64 time deltas. 12.39 +// Reflects an implementation detail inside lldeadmantimer.cpp 12.40 + 12.41 +static LLDeadmanTimer::time_type float_time_to_u64(F64 delta) 12.42 +{ 12.43 + return LLDeadmanTimer::time_type(delta * gClockFrequency); 12.44 +} 12.45 + 12.46 +static F64 u64_time_to_float(LLDeadmanTimer::time_type delta) 12.47 +{ 12.48 + return delta * gClockFrequencyInv; 12.49 +} 12.50 + 12.51 + 12.52 +namespace tut 12.53 +{ 12.54 + 12.55 +struct deadmantimer_test 12.56 +{ 12.57 + deadmantimer_test() 12.58 + { 12.59 + // LLTimer internals updating 12.60 + update_clock_frequencies(); 12.61 + } 12.62 +}; 12.63 + 12.64 +typedef test_group<deadmantimer_test> deadmantimer_group_t; 12.65 +typedef deadmantimer_group_t::object deadmantimer_object_t; 12.66 +tut::deadmantimer_group_t deadmantimer_instance("LLDeadmanTimer"); 12.67 + 12.68 +// Basic construction test and isExpired() call 12.69 +template<> template<> 12.70 +void deadmantimer_object_t::test<1>() 12.71 +{ 12.72 + { 12.73 + // Without cpu metrics 12.74 + F64 started(42.0), stopped(97.0); 12.75 + U64 count(U64L(8)); 12.76 + LLDeadmanTimer timer(10.0, false); 12.77 + 12.78 + ensure_equals("WOCM isExpired() returns false after ctor()", timer.isExpired(0, started, stopped, count), false); 12.79 + ensure_approximately_equals("WOCM t1 - isExpired() does not modify started", started, F64(42.0), 2); 12.80 + ensure_approximately_equals("WOCM t1 - isExpired() does not modify stopped", stopped, F64(97.0), 2); 12.81 + ensure_equals("WOCM t1 - isExpired() does not modify count", count, U64L(8)); 12.82 + } 12.83 + 12.84 + { 12.85 + // With cpu metrics 12.86 + F64 started(42.0), stopped(97.0); 12.87 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.88 + LLDeadmanTimer timer(10.0, true); 12.89 + 12.90 + ensure_equals("WCM isExpired() returns false after ctor()", timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); 12.91 + ensure_approximately_equals("WCM t1 - isExpired() does not modify started", started, F64(42.0), 2); 12.92 + ensure_approximately_equals("WCM t1 - isExpired() does not modify stopped", stopped, F64(97.0), 2); 12.93 + ensure_equals("WCM t1 - isExpired() does not modify count", count, U64L(8)); 12.94 + ensure_equals("WCM t1 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); 12.95 + ensure_equals("WCM t1 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); 12.96 + } 12.97 +} 12.98 + 12.99 + 12.100 +// Construct with zero horizon - not useful generally but will be useful in testing 12.101 +template<> template<> 12.102 +void deadmantimer_object_t::test<2>() 12.103 +{ 12.104 + { 12.105 + // Without cpu metrics 12.106 + F64 started(42.0), stopped(97.0); 12.107 + U64 count(U64L(8)); 12.108 + LLDeadmanTimer timer(0.0, false); // Zero is pre-expired 12.109 + 12.110 + ensure_equals("WOCM isExpired() still returns false with 0.0 time ctor()", 12.111 + timer.isExpired(0, started, stopped, count), false); 12.112 + } 12.113 + 12.114 + { 12.115 + // With cpu metrics 12.116 + F64 started(42.0), stopped(97.0); 12.117 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.118 + LLDeadmanTimer timer(0.0, true); // Zero is pre-expired 12.119 + 12.120 + ensure_equals("WCM isExpired() still returns false with 0.0 time ctor()", 12.121 + timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); 12.122 + } 12.123 +} 12.124 + 12.125 + 12.126 +// "pre-expired" timer - starting a timer with a 0.0 horizon will result in 12.127 +// expiration on first test. 12.128 +template<> template<> 12.129 +void deadmantimer_object_t::test<3>() 12.130 +{ 12.131 + { 12.132 + // Without cpu metrics 12.133 + F64 started(42.0), stopped(97.0); 12.134 + U64 count(U64L(8)); 12.135 + LLDeadmanTimer timer(0.0, false); 12.136 + 12.137 + timer.start(0); 12.138 + ensure_equals("WOCM isExpired() returns true with 0.0 horizon time", 12.139 + timer.isExpired(0, started, stopped, count), true); 12.140 + ensure_approximately_equals("WOCM expired timer with no bell ringing has stopped == started", started, stopped, 8); 12.141 + } 12.142 + { 12.143 + // With cpu metrics 12.144 + F64 started(42.0), stopped(97.0); 12.145 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.146 + LLDeadmanTimer timer(0.0, true); 12.147 + 12.148 + timer.start(0); 12.149 + ensure_equals("WCM isExpired() returns true with 0.0 horizon time", 12.150 + timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), true); 12.151 + ensure_approximately_equals("WCM expired timer with no bell ringing has stopped == started", started, stopped, 8); 12.152 + } 12.153 +} 12.154 + 12.155 + 12.156 +// "pre-expired" timer - bell rings are ignored as we're already expired. 12.157 +template<> template<> 12.158 +void deadmantimer_object_t::test<4>() 12.159 +{ 12.160 + { 12.161 + // Without cpu metrics 12.162 + F64 started(42.0), stopped(97.0); 12.163 + U64 count(U64L(8)); 12.164 + LLDeadmanTimer timer(0.0, false); 12.165 + 12.166 + timer.start(0); 12.167 + timer.ringBell(LLDeadmanTimer::getNow() + float_time_to_u64(1000.0), 1); 12.168 + ensure_equals("WOCM isExpired() returns true with 0.0 horizon time after bell ring", 12.169 + timer.isExpired(0, started, stopped, count), true); 12.170 + ensure_approximately_equals("WOCM ringBell has no impact on expired timer leaving stopped == started", started, stopped, 8); 12.171 + } 12.172 + { 12.173 + // With cpu metrics 12.174 + F64 started(42.0), stopped(97.0); 12.175 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.176 + LLDeadmanTimer timer(0.0, true); 12.177 + 12.178 + timer.start(0); 12.179 + timer.ringBell(LLDeadmanTimer::getNow() + float_time_to_u64(1000.0), 1); 12.180 + ensure_equals("WCM isExpired() returns true with 0.0 horizon time after bell ring", 12.181 + timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), true); 12.182 + ensure_approximately_equals("WCM ringBell has no impact on expired timer leaving stopped == started", started, stopped, 8); 12.183 + } 12.184 +} 12.185 + 12.186 + 12.187 +// start(0) test - unexpired timer reports unexpired 12.188 +template<> template<> 12.189 +void deadmantimer_object_t::test<5>() 12.190 +{ 12.191 + { 12.192 + // Without cpu metrics 12.193 + F64 started(42.0), stopped(97.0); 12.194 + U64 count(U64L(8)); 12.195 + LLDeadmanTimer timer(10.0, false); 12.196 + 12.197 + timer.start(0); 12.198 + ensure_equals("WOCM isExpired() returns false after starting with 10.0 horizon time", 12.199 + timer.isExpired(0, started, stopped, count), false); 12.200 + ensure_approximately_equals("WOCM t5 - isExpired() does not modify started", started, F64(42.0), 2); 12.201 + ensure_approximately_equals("WOCM t5 - isExpired() does not modify stopped", stopped, F64(97.0), 2); 12.202 + ensure_equals("WOCM t5 - isExpired() does not modify count", count, U64L(8)); 12.203 + } 12.204 + { 12.205 + // With cpu metrics 12.206 + F64 started(42.0), stopped(97.0); 12.207 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.208 + LLDeadmanTimer timer(10.0, true); 12.209 + 12.210 + timer.start(0); 12.211 + ensure_equals("WCM isExpired() returns false after starting with 10.0 horizon time", 12.212 + timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); 12.213 + ensure_approximately_equals("WCM t5 - isExpired() does not modify started", started, F64(42.0), 2); 12.214 + ensure_approximately_equals("WCM t5 - isExpired() does not modify stopped", stopped, F64(97.0), 2); 12.215 + ensure_equals("WCM t5 - isExpired() does not modify count", count, U64L(8)); 12.216 + ensure_equals("WCM t5 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); 12.217 + ensure_equals("WCM t5 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); 12.218 + } 12.219 +} 12.220 + 12.221 + 12.222 +// start() test - start in the past but not beyond 1 horizon 12.223 +template<> template<> 12.224 +void deadmantimer_object_t::test<6>() 12.225 +{ 12.226 + { 12.227 + // Without cpu metrics 12.228 + F64 started(42.0), stopped(97.0); 12.229 + U64 count(U64L(8)); 12.230 + LLDeadmanTimer timer(10.0, false); 12.231 + 12.232 + // Would like to do subtraction on current time but can't because 12.233 + // the implementation on Windows is zero-based. We wrap around 12.234 + // the backside resulting in a large U64 number. 12.235 + 12.236 + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); 12.237 + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(5.0)); 12.238 + timer.start(the_past); 12.239 + ensure_equals("WOCM t6 - isExpired() returns false with 10.0 horizon time starting 5.0 in past", 12.240 + timer.isExpired(now, started, stopped, count), false); 12.241 + ensure_approximately_equals("WOCM t6 - isExpired() does not modify started", started, F64(42.0), 2); 12.242 + ensure_approximately_equals("WOCM t6 - isExpired() does not modify stopped", stopped, F64(97.0), 2); 12.243 + ensure_equals("WOCM t6 - isExpired() does not modify count", count, U64L(8)); 12.244 + } 12.245 + { 12.246 + // With cpu metrics 12.247 + F64 started(42.0), stopped(97.0); 12.248 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.249 + LLDeadmanTimer timer(10.0, true); 12.250 + 12.251 + // Would like to do subtraction on current time but can't because 12.252 + // the implementation on Windows is zero-based. We wrap around 12.253 + // the backside resulting in a large U64 number. 12.254 + 12.255 + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); 12.256 + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(5.0)); 12.257 + timer.start(the_past); 12.258 + ensure_equals("WCM t6 - isExpired() returns false with 10.0 horizon time starting 5.0 in past", 12.259 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); 12.260 + ensure_approximately_equals("WCM t6 - isExpired() does not modify started", started, F64(42.0), 2); 12.261 + ensure_approximately_equals("WCM t6 - isExpired() does not modify stopped", stopped, F64(97.0), 2); 12.262 + ensure_equals("t6 - isExpired() does not modify count", count, U64L(8)); 12.263 + ensure_equals("WCM t6 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); 12.264 + ensure_equals("WCM t6 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); 12.265 + } 12.266 +} 12.267 + 12.268 + 12.269 +// start() test - start in the past but well beyond 1 horizon 12.270 +template<> template<> 12.271 +void deadmantimer_object_t::test<7>() 12.272 +{ 12.273 + { 12.274 + // Without cpu metrics 12.275 + F64 started(42.0), stopped(97.0); 12.276 + U64 count(U64L(8)); 12.277 + LLDeadmanTimer timer(10.0, false); 12.278 + 12.279 + // Would like to do subtraction on current time but can't because 12.280 + // the implementation on Windows is zero-based. We wrap around 12.281 + // the backside resulting in a large U64 number. 12.282 + 12.283 + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); 12.284 + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); 12.285 + timer.start(the_past); 12.286 + ensure_equals("WOCM t7 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", 12.287 + timer.isExpired(now,started, stopped, count), true); 12.288 + ensure_approximately_equals("WOCM t7 - starting before horizon still gives equal started / stopped", started, stopped, 8); 12.289 + } 12.290 + { 12.291 + // With cpu metrics 12.292 + F64 started(42.0), stopped(97.0); 12.293 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.294 + LLDeadmanTimer timer(10.0, true); 12.295 + 12.296 + // Would like to do subtraction on current time but can't because 12.297 + // the implementation on Windows is zero-based. We wrap around 12.298 + // the backside resulting in a large U64 number. 12.299 + 12.300 + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); 12.301 + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); 12.302 + timer.start(the_past); 12.303 + ensure_equals("WCM t7 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", 12.304 + timer.isExpired(now,started, stopped, count, user_cpu, sys_cpu), true); 12.305 + ensure_approximately_equals("WOCM t7 - starting before horizon still gives equal started / stopped", started, stopped, 8); 12.306 + } 12.307 +} 12.308 + 12.309 + 12.310 +// isExpired() test - results are read-once. Probes after first true are false. 12.311 +template<> template<> 12.312 +void deadmantimer_object_t::test<8>() 12.313 +{ 12.314 + { 12.315 + // Without cpu metrics 12.316 + F64 started(42.0), stopped(97.0); 12.317 + U64 count(U64L(8)); 12.318 + LLDeadmanTimer timer(10.0, false); 12.319 + 12.320 + // Would like to do subtraction on current time but can't because 12.321 + // the implementation on Windows is zero-based. We wrap around 12.322 + // the backside resulting in a large U64 number. 12.323 + 12.324 + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); 12.325 + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); 12.326 + timer.start(the_past); 12.327 + ensure_equals("WOCM t8 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", 12.328 + timer.isExpired(now, started, stopped, count), true); 12.329 + 12.330 + started = 42.0; 12.331 + stopped = 97.0; 12.332 + count = U64L(8); 12.333 + ensure_equals("WOCM t8 - second isExpired() returns false after true", 12.334 + timer.isExpired(now, started, stopped, count), false); 12.335 + ensure_approximately_equals("WOCM t8 - 2nd isExpired() does not modify started", started, F64(42.0), 2); 12.336 + ensure_approximately_equals("WOCM t8 - 2nd isExpired() does not modify stopped", stopped, F64(97.0), 2); 12.337 + ensure_equals("WOCM t8 - 2nd isExpired() does not modify count", count, U64L(8)); 12.338 + } 12.339 + { 12.340 + // With cpu metrics 12.341 + F64 started(42.0), stopped(97.0); 12.342 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.343 + LLDeadmanTimer timer(10.0, true); 12.344 + 12.345 + // Would like to do subtraction on current time but can't because 12.346 + // the implementation on Windows is zero-based. We wrap around 12.347 + // the backside resulting in a large U64 number. 12.348 + 12.349 + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); 12.350 + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); 12.351 + timer.start(the_past); 12.352 + ensure_equals("WCM t8 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", 12.353 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); 12.354 + 12.355 + started = 42.0; 12.356 + stopped = 97.0; 12.357 + count = U64L(8); 12.358 + user_cpu = 29000; 12.359 + sys_cpu = 57000; 12.360 + ensure_equals("WCM t8 - second isExpired() returns false after true", 12.361 + timer.isExpired(now, started, stopped, count), false); 12.362 + ensure_approximately_equals("WCM t8 - 2nd isExpired() does not modify started", started, F64(42.0), 2); 12.363 + ensure_approximately_equals("WCM t8 - 2nd isExpired() does not modify stopped", stopped, F64(97.0), 2); 12.364 + ensure_equals("WCM t8 - 2nd isExpired() does not modify count", count, U64L(8)); 12.365 + ensure_equals("WCM t8 - 2nd isExpired() does not modify user_cpu", user_cpu, U64L(29000)); 12.366 + ensure_equals("WCM t8 - 2nd isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); 12.367 + } 12.368 +} 12.369 + 12.370 + 12.371 +// ringBell() test - see that we can keep a timer from expiring 12.372 +template<> template<> 12.373 +void deadmantimer_object_t::test<9>() 12.374 +{ 12.375 + { 12.376 + // Without cpu metrics 12.377 + F64 started(42.0), stopped(97.0); 12.378 + U64 count(U64L(8)); 12.379 + LLDeadmanTimer timer(5.0, false); 12.380 + 12.381 + LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); 12.382 + F64 real_start(u64_time_to_float(now)); 12.383 + timer.start(0); 12.384 + 12.385 + now += float_time_to_u64(1.0); 12.386 + timer.ringBell(now, 1); 12.387 + now += float_time_to_u64(1.0); 12.388 + timer.ringBell(now, 1); 12.389 + now += float_time_to_u64(1.0); 12.390 + timer.ringBell(now, 1); 12.391 + now += float_time_to_u64(1.0); 12.392 + timer.ringBell(now, 1); 12.393 + now += float_time_to_u64(1.0); 12.394 + timer.ringBell(now, 1); 12.395 + now += float_time_to_u64(1.0); 12.396 + timer.ringBell(now, 1); 12.397 + now += float_time_to_u64(1.0); 12.398 + timer.ringBell(now, 1); 12.399 + now += float_time_to_u64(1.0); 12.400 + timer.ringBell(now, 1); 12.401 + now += float_time_to_u64(1.0); 12.402 + timer.ringBell(now, 1); 12.403 + now += float_time_to_u64(1.0); 12.404 + timer.ringBell(now, 1); 12.405 + ensure_equals("WOCM t9 - 5.0 horizon timer has not timed out after 10 1-second bell rings", 12.406 + timer.isExpired(now, started, stopped, count), false); 12.407 + F64 last_good_ring(u64_time_to_float(now)); 12.408 + 12.409 + // Jump forward and expire 12.410 + now += float_time_to_u64(10.0); 12.411 + ensure_equals("WOCM t9 - 5.0 horizon timer expires on 10-second jump", 12.412 + timer.isExpired(now, started, stopped, count), true); 12.413 + ensure_approximately_equals("WOCM t9 - started matches start() time", started, real_start, 4); 12.414 + ensure_approximately_equals("WOCM t9 - stopped matches last ringBell() time", stopped, last_good_ring, 4); 12.415 + ensure_equals("WOCM t9 - 10 good ringBell()s", count, U64L(10)); 12.416 + ensure_equals("WOCM t9 - single read only", timer.isExpired(now, started, stopped, count), false); 12.417 + } 12.418 + { 12.419 + // With cpu metrics 12.420 + F64 started(42.0), stopped(97.0); 12.421 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.422 + LLDeadmanTimer timer(5.0, true); 12.423 + 12.424 + LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); 12.425 + F64 real_start(u64_time_to_float(now)); 12.426 + timer.start(0); 12.427 + 12.428 + now += float_time_to_u64(1.0); 12.429 + timer.ringBell(now, 1); 12.430 + now += float_time_to_u64(1.0); 12.431 + timer.ringBell(now, 1); 12.432 + now += float_time_to_u64(1.0); 12.433 + timer.ringBell(now, 1); 12.434 + now += float_time_to_u64(1.0); 12.435 + timer.ringBell(now, 1); 12.436 + now += float_time_to_u64(1.0); 12.437 + timer.ringBell(now, 1); 12.438 + now += float_time_to_u64(1.0); 12.439 + timer.ringBell(now, 1); 12.440 + now += float_time_to_u64(1.0); 12.441 + timer.ringBell(now, 1); 12.442 + now += float_time_to_u64(1.0); 12.443 + timer.ringBell(now, 1); 12.444 + now += float_time_to_u64(1.0); 12.445 + timer.ringBell(now, 1); 12.446 + now += float_time_to_u64(1.0); 12.447 + timer.ringBell(now, 1); 12.448 + ensure_equals("WCM t9 - 5.0 horizon timer has not timed out after 10 1-second bell rings", 12.449 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); 12.450 + F64 last_good_ring(u64_time_to_float(now)); 12.451 + 12.452 + // Jump forward and expire 12.453 + now += float_time_to_u64(10.0); 12.454 + ensure_equals("WCM t9 - 5.0 horizon timer expires on 10-second jump", 12.455 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); 12.456 + ensure_approximately_equals("WCM t9 - started matches start() time", started, real_start, 4); 12.457 + ensure_approximately_equals("WCM t9 - stopped matches last ringBell() time", stopped, last_good_ring, 4); 12.458 + ensure_equals("WCM t9 - 10 good ringBell()s", count, U64L(10)); 12.459 + ensure_equals("WCM t9 - single read only", timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); 12.460 + } 12.461 +} 12.462 + 12.463 + 12.464 +// restart after expiration test - verify that restarts behave well 12.465 +template<> template<> 12.466 +void deadmantimer_object_t::test<10>() 12.467 +{ 12.468 + { 12.469 + // Without cpu metrics 12.470 + F64 started(42.0), stopped(97.0); 12.471 + U64 count(U64L(8)); 12.472 + LLDeadmanTimer timer(5.0, false); 12.473 + 12.474 + LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); 12.475 + F64 real_start(u64_time_to_float(now)); 12.476 + timer.start(0); 12.477 + 12.478 + now += float_time_to_u64(1.0); 12.479 + timer.ringBell(now, 1); 12.480 + now += float_time_to_u64(1.0); 12.481 + timer.ringBell(now, 1); 12.482 + now += float_time_to_u64(1.0); 12.483 + timer.ringBell(now, 1); 12.484 + now += float_time_to_u64(1.0); 12.485 + timer.ringBell(now, 1); 12.486 + now += float_time_to_u64(1.0); 12.487 + timer.ringBell(now, 1); 12.488 + now += float_time_to_u64(1.0); 12.489 + timer.ringBell(now, 1); 12.490 + now += float_time_to_u64(1.0); 12.491 + timer.ringBell(now, 1); 12.492 + now += float_time_to_u64(1.0); 12.493 + timer.ringBell(now, 1); 12.494 + now += float_time_to_u64(1.0); 12.495 + timer.ringBell(now, 1); 12.496 + now += float_time_to_u64(1.0); 12.497 + timer.ringBell(now, 1); 12.498 + ensure_equals("WOCM t10 - 5.0 horizon timer has not timed out after 10 1-second bell rings", 12.499 + timer.isExpired(now, started, stopped, count), false); 12.500 + F64 last_good_ring(u64_time_to_float(now)); 12.501 + 12.502 + // Jump forward and expire 12.503 + now += float_time_to_u64(10.0); 12.504 + ensure_equals("WOCM t10 - 5.0 horizon timer expires on 10-second jump", 12.505 + timer.isExpired(now, started, stopped, count), true); 12.506 + ensure_approximately_equals("WOCM t10 - started matches start() time", started, real_start, 4); 12.507 + ensure_approximately_equals("WOCM t10 - stopped matches last ringBell() time", stopped, last_good_ring, 4); 12.508 + ensure_equals("WOCM t10 - 10 good ringBell()s", count, U64L(10)); 12.509 + ensure_equals("WOCM t10 - single read only", timer.isExpired(now, started, stopped, count), false); 12.510 + 12.511 + // Jump forward and restart 12.512 + now += float_time_to_u64(1.0); 12.513 + real_start = u64_time_to_float(now); 12.514 + timer.start(now); 12.515 + 12.516 + // Run a modified bell ring sequence 12.517 + now += float_time_to_u64(1.0); 12.518 + timer.ringBell(now, 1); 12.519 + now += float_time_to_u64(1.0); 12.520 + timer.ringBell(now, 1); 12.521 + now += float_time_to_u64(1.0); 12.522 + timer.ringBell(now, 1); 12.523 + now += float_time_to_u64(1.0); 12.524 + timer.ringBell(now, 1); 12.525 + now += float_time_to_u64(1.0); 12.526 + timer.ringBell(now, 1); 12.527 + now += float_time_to_u64(1.0); 12.528 + timer.ringBell(now, 1); 12.529 + now += float_time_to_u64(1.0); 12.530 + timer.ringBell(now, 1); 12.531 + now += float_time_to_u64(1.0); 12.532 + timer.ringBell(now, 1); 12.533 + ensure_equals("WOCM t10 - 5.0 horizon timer has not timed out after 8 1-second bell rings", 12.534 + timer.isExpired(now, started, stopped, count), false); 12.535 + last_good_ring = u64_time_to_float(now); 12.536 + 12.537 + // Jump forward and expire 12.538 + now += float_time_to_u64(10.0); 12.539 + ensure_equals("WOCM t10 - 5.0 horizon timer expires on 8-second jump", 12.540 + timer.isExpired(now, started, stopped, count), true); 12.541 + ensure_approximately_equals("WOCM t10 - 2nd started matches start() time", started, real_start, 4); 12.542 + ensure_approximately_equals("WOCM t10 - 2nd stopped matches last ringBell() time", stopped, last_good_ring, 4); 12.543 + ensure_equals("WOCM t10 - 8 good ringBell()s", count, U64L(8)); 12.544 + ensure_equals("WOCM t10 - single read only - 2nd start", 12.545 + timer.isExpired(now, started, stopped, count), false); 12.546 + } 12.547 + { 12.548 + // With cpu metrics 12.549 + F64 started(42.0), stopped(97.0); 12.550 + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); 12.551 + 12.552 + LLDeadmanTimer timer(5.0, true); 12.553 + 12.554 + LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); 12.555 + F64 real_start(u64_time_to_float(now)); 12.556 + timer.start(0); 12.557 + 12.558 + now += float_time_to_u64(1.0); 12.559 + timer.ringBell(now, 1); 12.560 + now += float_time_to_u64(1.0); 12.561 + timer.ringBell(now, 1); 12.562 + now += float_time_to_u64(1.0); 12.563 + timer.ringBell(now, 1); 12.564 + now += float_time_to_u64(1.0); 12.565 + timer.ringBell(now, 1); 12.566 + now += float_time_to_u64(1.0); 12.567 + timer.ringBell(now, 1); 12.568 + now += float_time_to_u64(1.0); 12.569 + timer.ringBell(now, 1); 12.570 + now += float_time_to_u64(1.0); 12.571 + timer.ringBell(now, 1); 12.572 + now += float_time_to_u64(1.0); 12.573 + timer.ringBell(now, 1); 12.574 + now += float_time_to_u64(1.0); 12.575 + timer.ringBell(now, 1); 12.576 + now += float_time_to_u64(1.0); 12.577 + timer.ringBell(now, 1); 12.578 + ensure_equals("WCM t10 - 5.0 horizon timer has not timed out after 10 1-second bell rings", 12.579 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); 12.580 + F64 last_good_ring(u64_time_to_float(now)); 12.581 + 12.582 + // Jump forward and expire 12.583 + now += float_time_to_u64(10.0); 12.584 + ensure_equals("WCM t10 - 5.0 horizon timer expires on 10-second jump", 12.585 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); 12.586 + ensure_approximately_equals("WCM t10 - started matches start() time", started, real_start, 4); 12.587 + ensure_approximately_equals("WCM t10 - stopped matches last ringBell() time", stopped, last_good_ring, 4); 12.588 + ensure_equals("WCM t10 - 10 good ringBell()s", count, U64L(10)); 12.589 + ensure_equals("WCM t10 - single read only", timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); 12.590 + 12.591 + // Jump forward and restart 12.592 + now += float_time_to_u64(1.0); 12.593 + real_start = u64_time_to_float(now); 12.594 + timer.start(now); 12.595 + 12.596 + // Run a modified bell ring sequence 12.597 + now += float_time_to_u64(1.0); 12.598 + timer.ringBell(now, 1); 12.599 + now += float_time_to_u64(1.0); 12.600 + timer.ringBell(now, 1); 12.601 + now += float_time_to_u64(1.0); 12.602 + timer.ringBell(now, 1); 12.603 + now += float_time_to_u64(1.0); 12.604 + timer.ringBell(now, 1); 12.605 + now += float_time_to_u64(1.0); 12.606 + timer.ringBell(now, 1); 12.607 + now += float_time_to_u64(1.0); 12.608 + timer.ringBell(now, 1); 12.609 + now += float_time_to_u64(1.0); 12.610 + timer.ringBell(now, 1); 12.611 + now += float_time_to_u64(1.0); 12.612 + timer.ringBell(now, 1); 12.613 + ensure_equals("WCM t10 - 5.0 horizon timer has not timed out after 8 1-second bell rings", 12.614 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); 12.615 + last_good_ring = u64_time_to_float(now); 12.616 + 12.617 + // Jump forward and expire 12.618 + now += float_time_to_u64(10.0); 12.619 + ensure_equals("WCM t10 - 5.0 horizon timer expires on 8-second jump", 12.620 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); 12.621 + ensure_approximately_equals("WCM t10 - 2nd started matches start() time", started, real_start, 4); 12.622 + ensure_approximately_equals("WCM t10 - 2nd stopped matches last ringBell() time", stopped, last_good_ring, 4); 12.623 + ensure_equals("WCM t10 - 8 good ringBell()s", count, U64L(8)); 12.624 + ensure_equals("WCM t10 - single read only - 2nd start", 12.625 + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); 12.626 + } 12.627 +} 12.628 + 12.629 + 12.630 + 12.631 +} // end namespace tut
13.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 13.2 +++ b/indra/llcommon/tests/llprocinfo_test.cpp Tue Feb 25 13:25:40 2014 -0500 13.3 @@ -0,0 +1,91 @@ 13.4 +/** 13.5 + * @file llprocinfo_test.cpp 13.6 + * @brief Tests for the LLProcInfo class. 13.7 + * 13.8 + * $LicenseInfo:firstyear=2013&license=viewerlgpl$ 13.9 + * Second Life Viewer Source Code 13.10 + * Copyright (C) 2013, Linden Research, Inc. 13.11 + * 13.12 + * This library is free software; you can redistribute it and/or 13.13 + * modify it under the terms of the GNU Lesser General Public 13.14 + * License as published by the Free Software Foundation; 13.15 + * version 2.1 of the License only. 13.16 + * 13.17 + * This library is distributed in the hope that it will be useful, 13.18 + * but WITHOUT ANY WARRANTY; without even the implied warranty of 13.19 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13.20 + * Lesser General Public License for more details. 13.21 + * 13.22 + * You should have received a copy of the GNU Lesser General Public 13.23 + * License along with this library; if not, write to the Free Software 13.24 + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 13.25 + * 13.26 + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA 13.27 + * $/LicenseInfo$ 13.28 + */ 13.29 + 13.30 +#include "linden_common.h" 13.31 + 13.32 +#include "../llprocinfo.h" 13.33 + 13.34 +#include "../test/lltut.h" 13.35 +#include "../lltimer.h" 13.36 + 13.37 + 13.38 +static const LLProcInfo::time_type bad_user(289375U), bad_system(275U); 13.39 + 13.40 + 13.41 +namespace tut 13.42 +{ 13.43 + 13.44 +struct procinfo_test 13.45 +{ 13.46 + procinfo_test() 13.47 + { 13.48 + } 13.49 +}; 13.50 + 13.51 +typedef test_group<procinfo_test> procinfo_group_t; 13.52 +typedef procinfo_group_t::object procinfo_object_t; 13.53 +tut::procinfo_group_t procinfo_instance("LLProcInfo"); 13.54 + 13.55 + 13.56 +// Basic invocation works 13.57 +template<> template<> 13.58 +void procinfo_object_t::test<1>() 13.59 +{ 13.60 + LLProcInfo::time_type user(bad_user), system(bad_system); 13.61 + 13.62 + set_test_name("getCPUUsage() basic function"); 13.63 + 13.64 + LLProcInfo::getCPUUsage(user, system); 13.65 + 13.66 + ensure_not_equals("getCPUUsage() writes to its user argument", user, bad_user); 13.67 + ensure_not_equals("getCPUUsage() writes to its system argument", system, bad_system); 13.68 +} 13.69 + 13.70 + 13.71 +// Time increases 13.72 +template<> template<> 13.73 +void procinfo_object_t::test<2>() 13.74 +{ 13.75 + LLProcInfo::time_type user(bad_user), system(bad_system); 13.76 + LLProcInfo::time_type user2(bad_user), system2(bad_system); 13.77 + 13.78 + set_test_name("getCPUUsage() increases over time"); 13.79 + 13.80 + LLProcInfo::getCPUUsage(user, system); 13.81 + 13.82 + for (int i(0); i < 100000; ++i) 13.83 + { 13.84 + ms_sleep(0); 13.85 + } 13.86 + 13.87 + LLProcInfo::getCPUUsage(user2, system2); 13.88 + 13.89 + ensure_equals("getCPUUsage() user value doesn't decrease over time", user2 >= user, true); 13.90 + ensure_equals("getCPUUsage() system value doesn't decrease over time", system2 >= system, true); 13.91 +} 13.92 + 13.93 + 13.94 +} // end namespace tut
14.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 14.2 +++ b/indra/llcorehttp/README.Linden Tue Feb 25 13:25:40 2014 -0500 14.3 @@ -0,0 +1,671 @@ 14.4 + 14.5 + 14.6 + 14.7 +1. HTTP Fetching in 15 Minutes 14.8 + 14.9 + Let's start with a trivial working example. You'll need a throwaway 14.10 + build of the viewer. And we'll use indra/newview/llappviewer.cpp as 14.11 + the host module for these hacks. 14.12 + 14.13 + First, add some headers: 14.14 + 14.15 + 14.16 + #include "httpcommon.h" 14.17 + #include "httprequest.h" 14.18 + #include "httphandler.h" 14.19 + 14.20 + 14.21 + You'll need to derive a class from HttpHandler (not HttpHandle). 14.22 + This is used to deliver notifications of HTTP completion to your 14.23 + code. Place it near the top, before LLDeferredTaskList, say: 14.24 + 14.25 + 14.26 + class MyHandler : public LLCore::HttpHandler 14.27 + { 14.28 + public: 14.29 + MyHandler() 14.30 + : LLCore::HttpHandler() 14.31 + {} 14.32 + 14.33 + virtual void onCompleted(LLCore::HttpHandle /* handle */, 14.34 + LLCore::HttpResponse * /* response */) 14.35 + { 14.36 + LL_INFOS("Hack") << "It is happening again." << LL_ENDL; 14.37 + 14.38 + delete this; // Last statement 14.39 + } 14.40 + }; 14.41 + 14.42 + 14.43 + Add some statics up there as well: 14.44 + 14.45 + 14.46 + // Our request object. Allocate during initialiation. 14.47 + static LLCore::HttpRequest * my_request(NULL); 14.48 + 14.49 + // The policy class for HTTP traffic. 14.50 + // Use HttpRequest::DEFAULT_POLICY_ID, but DO NOT SHIP WITH THIS VALUE!! 14.51 + static LLCore::HttpRequest::policy_t my_policy(LLCore::HttpRequest::DEFAULT_POLICY_ID); 14.52 + 14.53 + // Priority for HTTP requests. Use 0U. 14.54 + static LLCore::HttpRequest::priority_t my_priority(0U); 14.55 + 14.56 + 14.57 + In LLAppViewer::init() after mAppCoreHttp.init(), create a request object: 14.58 + 14.59 + 14.60 + my_request = new LLCore::HttpRequest(); 14.61 + 14.62 + 14.63 + In LLAppViewer::mainLoop(), just before entering the while loop, 14.64 + we'll kick off one HTTP request: 14.65 + 14.66 + 14.67 + // Construct a handler object (we'll use the heap this time): 14.68 + MyHandler * my_handler = new MyHandler; 14.69 + 14.70 + // Issue a GET request to 'http://www.example.com/' kicking off 14.71 + // all the I/O, retry logic, etc. 14.72 + LLCore::HttpHandle handle; 14.73 + handle = my_request->requestGet(my_policy, 14.74 + my_priority, 14.75 + "http://www.example.com/", 14.76 + NULL, 14.77 + NULL, 14.78 + my_handler); 14.79 + if (LLCORE_HTTP_HANDLE_INVALID == handle) 14.80 + { 14.81 + LL_WARNS("Hack") << "Failed to launch HTTP request. Try again." 14.82 + << LL_ENDL; 14.83 + } 14.84 + 14.85 + 14.86 + Finally, arrange to periodically call update() on the request object 14.87 + to find out when the request completes. This will be done by 14.88 + calling the onCompleted() method with status information and 14.89 + response data from the HTTP operation. Add this to the 14.90 + LLAppViewer::idle() method after the ping: 14.91 + 14.92 + 14.93 + my_request->update(0); 14.94 + 14.95 + 14.96 + That's it. Build it, run it and watch the log file. You should get 14.97 + the "It is happening again." message indicating that the HTTP 14.98 + operation completed in some manner. 14.99 + 14.100 + 14.101 +2. What Does All That Mean 14.102 + 14.103 + MyHandler/HttpHandler. This class replaces the Responder-style in 14.104 + legacy code. One method is currently defined. It is used for all 14.105 + request completions, successful or failed: 14.106 + 14.107 + 14.108 + void onCompleted(LLCore::HttpHandle /* handle */, 14.109 + LLCore::HttpResponse * /* response */); 14.110 + 14.111 + 14.112 + The onCompleted() method is invoked as a callback during calls to 14.113 + HttpRequest::update(). All I/O is completed asynchronously in 14.114 + another thread. But notifications are polled by calling update() 14.115 + and invoking a handler for completed requests. 14.116 + 14.117 + In this example, the invocation also deletes the handler (which is 14.118 + never referenced by the llcorehttp code again). But other 14.119 + allocation models are possible including handlers shared by many 14.120 + requests, stack-based handlers and handlers mixed in with other, 14.121 + unrelated classes. 14.122 + 14.123 + LLCore::HttpRequest(). Instances of this class are used to request 14.124 + all major functions of the library. Initialization, starting 14.125 + requests, delivering final notification of completion and various 14.126 + utility operations are all done via instances. There is one very 14.127 + important rule for instances: 14.128 + 14.129 + Request objects may NOT be shared between threads. 14.130 + 14.131 + my_priority. The APIs support the idea of priority ordering of 14.132 + requests but it hasn't been implemented and the hope is that this 14.133 + will become useless and removed from the interface. Use 0U except 14.134 + as noted. 14.135 + 14.136 + my_policy. This is an important one. This library attempts to 14.137 + manage TCP connection usage more rigorously than in the past. This 14.138 + is done by issuing requests to a queue that has various settable 14.139 + properties. These establish connection usage for the queue as well 14.140 + as how queues compete with one another. (This is patterned after 14.141 + class-based queueing used in various networking stacks.) Several 14.142 + classes are pre-defined. Deciding when to use an existing class and 14.143 + when to create a new one will determine what kind of experience 14.144 + users have. We'll pick up this question in detail below. 14.145 + 14.146 + requestGet(). Issues an ordinary HTTP GET request to a given URL 14.147 + and associating the request with a policy class, a priority and an 14.148 + response handler. Two additional arguments, not used here, allow 14.149 + for additional headers on the request and for per-request options. 14.150 + If successful, the call returns a handle whose value is other than 14.151 + LLCORE_HTTP_HANDLE_INVALID. The HTTP operation is then performed 14.152 + asynchronously by another thread without any additional work by the 14.153 + caller. If the handle returned is invalid, you can get the status 14.154 + code by calling my_request->getStatus(). 14.155 + 14.156 + update(). To get notification that the request has completed, a 14.157 + call to update() will invoke onCompleted() methods. 14.158 + 14.159 + 14.160 +3. Refinements, Necessary and Otherwise 14.161 + 14.162 + MyHandler::onCompleted(). You'll want to do something useful with 14.163 + your response. Distinguish errors from successes and getting the 14.164 + response body back in some form. 14.165 + 14.166 + Add a new header: 14.167 + 14.168 + 14.169 + #include "bufferarray.h" 14.170 + 14.171 + 14.172 + Replace the existing MyHandler::onCompleted() definition with: 14.173 + 14.174 + 14.175 + virtual void onCompleted(LLCore::HttpHandle /* handle */, 14.176 + LLCore::HttpResponse * response) 14.177 + { 14.178 + LLCore::HttpStatus status = response->getStatus(); 14.179 + if (status) 14.180 + { 14.181 + // Successful request. Try to fetch the data 14.182 + LLCore::BufferArray * data = response->getBody(); 14.183 + 14.184 + if (data && data->size()) 14.185 + { 14.186 + // There's some data. A BufferArray is a linked list 14.187 + // of buckets. We'll create a linear buffer and copy 14.188 + // the data into it. 14.189 + size_t data_len = data->size(); 14.190 + char * data_blob = new char [data_len + 1]; 14.191 + data->read(0, data_blob, data_len); 14.192 + data_blob[data_len] = '\0'; 14.193 + 14.194 + // Process the data now in NUL-terminated string. 14.195 + // Needs more scrubbing but this will do. 14.196 + LL_INFOS("Hack") << "Received: " << data_blob << LL_ENDL; 14.197 + 14.198 + // Free the temporary data 14.199 + delete [] data_blob; 14.200 + } 14.201 + } 14.202 + else 14.203 + { 14.204 + // Something went wrong. Translate the status to 14.205 + // a meaningful message. 14.206 + LL_WARNS("Hack") << "HTTP GET failed. Status: " 14.207 + << status.toTerseString() 14.208 + << ", Reason: " << status.toString() 14.209 + << LL_ENDL; 14.210 + } 14.211 + 14.212 + delete this; // Last statement 14.213 + } 14.214 + 14.215 + 14.216 + HttpHeaders. The header file "httprequest.h" documents the expected 14.217 + important headers that will go out with the request. You can add to 14.218 + these by including an HttpHeaders object with the requestGet() call. 14.219 + These are typically setup once as part of init rather than 14.220 + dynamically created. 14.221 + 14.222 + Add another header: 14.223 + 14.224 + 14.225 + #include "httpheaders.h" 14.226 + 14.227 + 14.228 + In LLAppViewer::mainLoop(), add this alongside the allocation of 14.229 + my_handler: 14.230 + 14.231 + 14.232 + // Additional headers for all requests 14.233 + LLCore::HttpHeaders * my_headers = new LLCore::HttpHeaders(); 14.234 + my_headers->append("Accept", "text/html, application/llsd+xml"); 14.235 + 14.236 + 14.237 + HttpOptions. Options are similar and include a mix of value types. 14.238 + One interesting per-request option is the trace setting. This 14.239 + enables various debug-type messages in the log file that show the 14.240 + progress of the request through the library. It takes values from 14.241 + zero to three with higher values giving more verbose logging. We'll 14.242 + use '2' and this will also give us a chance to verify that 14.243 + HttpHeaders works as expected. 14.244 + 14.245 + Same as above, a new header: 14.246 + 14.247 + 14.248 + #include "httpoptions.h" 14.249 + 14.250 + 14.251 + And in LLAppView::mainLoop(): 14.252 + 14.253 + 14.254 + // Special options for requests 14.255 + LLCore::HttpOptions * my_options = new LLCore::HttpOptions(); 14.256 + my_options->setTrace(2); 14.257 + 14.258 + 14.259 + Now let's put that all together into a more complete requesting 14.260 + sequence. Replace the existing invocation of requestGet() with this 14.261 + slightly more elaborate block: 14.262 + 14.263 + 14.264 + LLCore::HttpHandle handle; 14.265 + handle = my_request->requestGet(my_policy, 14.266 + my_priority, 14.267 + "http://www.example.com/", 14.268 + my_options, 14.269 + my_headers, 14.270 + my_handler); 14.271 + if (LLCORE_HTTP_HANDLE_INVALID == handle) 14.272 + { 14.273 + LLCore::HttpStatus status = my_request->getStatus(); 14.274 + 14.275 + LL_WARNS("Hack") << "Failed to request HTTP GET. Status: " 14.276 + << status.toTerseString() 14.277 + << ", Reason: " << status.toString() 14.278 + << LL_ENDL; 14.279 + 14.280 + delete my_handler; // No longer needed. 14.281 + my_handler = NULL; 14.282 + } 14.283 + 14.284 + 14.285 + Build, run and examine the log file. You'll get some new data with 14.286 + this run. First, you should get the www.example.com home page 14.287 + content: 14.288 + 14.289 + 14.290 +---------------------------------------------------------------------------- 14.291 +2013-09-17T20:26:51Z INFO: MyHandler::onCompleted: Received: <!doctype html> 14.292 +<html> 14.293 +<head> 14.294 + <title>Example Domain</title> 14.295 + 14.296 + <meta charset="utf-8" /> 14.297 + <meta http-equiv="Content-type" content="text/html; charset=utf-8" /> 14.298 + <meta name="viewport" content="width=device-width, initial-scale=1" /> 14.299 + <style type="text/css"> 14.300 + body { 14.301 + background-color: #f0f0f2; 14.302 + margin: 0; 14.303 + padding: 0; 14.304 + font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; 14.305 + 14.306 + } 14.307 + div { 14.308 + width: 600px; 14.309 + margin: 5em auto; 14.310 + padding: 50px; 14.311 + background-color: #fff; 14.312 + border-radius: 1em; 14.313 + } 14.314 + a:link, a:visited { 14.315 + color: #38488f; 14.316 + text-decoration: none; 14.317 + } 14.318 + @media (max-width: 700px) { 14.319 + body { 14.320 + background-color: #fff; 14.321 + } 14.322 + div { 14.323 + width: auto; 14.324 + margin: 0 auto; 14.325 + border-radius: 0; 14.326 + padding: 1em; 14.327 + } 14.328 + } 14.329 + </style> 14.330 +</head> 14.331 + 14.332 +<body> 14.333 +<div> 14.334 + <h1>Example Domain</h1> 14.335 + <p>This domain is established to be used for illustrative examples in documents. You may use this 14.336 + domain in examples without prior coordination or asking for permission.</p> 14.337 + <p><a href="http://www.iana.org/domains/example">More information...</a></p> 14.338 +</div> 14.339 +</body> 14.340 +</html> 14.341 +---------------------------------------------------------------------------- 14.342 + 14.343 + 14.344 + You'll also get a detailed trace of the HTTP operation itself. Note 14.345 + the HEADEROUT line which shows the additional header added to the 14.346 + request. 14.347 + 14.348 + 14.349 +---------------------------------------------------------------------------- 14.350 +HttpService::processRequestQueue: TRACE, FromRequestQueue, Handle: 086D3148 14.351 +HttpLibcurl::addOp: TRACE, ToActiveQueue, Handle: 086D3148, Actives: 0, Readies: 0 14.352 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: About to connect() to www.example.com port 80 (#0) 14.353 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Trying 93.184.216.119... 14.354 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0) 14.355 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0) 14.356 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADEROUT, Data: GET / HTTP/1.1 Host: www.example.com Accept-Encoding: deflate, gzip Connection: keep-alive Keep-alive: 300 Accept: text/html, application/llsd+xml 14.357 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: HTTP/1.1 200 OK 14.358 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Accept-Ranges: bytes 14.359 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Cache-Control: max-age=604800 14.360 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Type: text/html 14.361 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Date: Tue, 17 Sep 2013 20:26:56 GMT 14.362 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Etag: "3012602696" 14.363 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Expires: Tue, 24 Sep 2013 20:26:56 GMT 14.364 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT 14.365 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Server: ECS (ewr/1590) 14.366 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: X-Cache: HIT 14.367 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: x-ec-custom-error: 1 14.368 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Length: 1270 14.369 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: 14.370 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: DATAIN, Data: 256 Bytes 14.371 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connection #0 to host www.example.com left intact 14.372 +HttpLibcurl::completeRequest: TRACE, RequestComplete, Handle: 086D3148, Status: Http_200 14.373 +HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 14.374 +---------------------------------------------------------------------------- 14.375 + 14.376 + 14.377 +4. What Does All That Mean, Part 2 14.378 + 14.379 + HttpStatus. The HttpStatus object encodes errors from libcurl, the 14.380 + library itself and HTTP status values. It does this to avoid 14.381 + collapsing all non-HTTP error into a single '499' HTTP status and to 14.382 + make errors distinct. 14.383 + 14.384 + To aid programming, the usual bool conversions are available so that 14.385 + you can write 'if (status)' and the expected thing will happen 14.386 + whether it's an HTTP, libcurl or library error. There's also 14.387 + provision to override the treatment of HTTP errors (making 404 a 14.388 + success, say). 14.389 + 14.390 + Share data, don't copy it. The library was started with the goal of 14.391 + avoiding data copies as much as possible. Instead, read-only data 14.392 + sharing across threads with atomic reference counts is used for a 14.393 + number of data types. These currently are: 14.394 + 14.395 + * BufferArray. Linked list of data blocks/HTTP bodies. 14.396 + * HttpHeaders. Shared headers for both requests and responses. 14.397 + * HttpOptions. Request-only data modifying HTTP behavior. 14.398 + * HttpResponse. HTTP response description given to onCompleted. 14.399 + 14.400 + Using objects of these types requires a few rules: 14.401 + 14.402 + * Constructor always gives a reference to caller. 14.403 + * References are dropped with release() not delete. 14.404 + * Additional references may be taken out with addRef(). 14.405 + * Unless otherwise stated, once an object is shared with another 14.406 + thread it should be treated as read-only. There's no 14.407 + synchronization on the objects themselves. 14.408 + 14.409 + HttpResponse. You'll encounter this mainly in onCompleted() methods. 14.410 + Commonly-used interfaces on this object: 14.411 + 14.412 + * getStatus() to return the final status of the request. 14.413 + * getBody() to retrieve the response body which may be NULL or 14.414 + zero-length. 14.415 + * getContentType() to return the value of the 'Content-Type' 14.416 + header or an empty string if none was sent. 14.417 + 14.418 + This is a reference-counted object so you can call addRef() on it 14.419 + and hold onto the response for an arbitrary time. But you'll 14.420 + usually just call a few methods and return from onCompleted() whose 14.421 + caller will release the object. 14.422 + 14.423 + BufferArray. The core data representation for request and response 14.424 + bodies. In HTTP responses, it's fetched with the getBody() method 14.425 + and may be NULL or non-NULL with zero length. All successful data 14.426 + handling should check both conditions before attempting to fetch 14.427 + data from the object. Data access model uses simple read/write 14.428 + semantics: 14.429 + 14.430 + * append() 14.431 + * size() 14.432 + * read() 14.433 + * write() 14.434 + 14.435 + (There is a more sophisticated stream adapter that extends these 14.436 + methods and will be covered below.) So, one way to retrieve data 14.437 + from a request is as follows: 14.438 + 14.439 + 14.440 + LLCore::BufferArray * data = response->getBody(); 14.441 + if (data && data->size()) 14.442 + { 14.443 + size_t data_len = data->size(); 14.444 + char * data_blob = new char [data_len + 1]; 14.445 + data->read(0, data_blob, data_len); 14.446 + 14.447 + 14.448 + HttpOptions and HttpResponse. Really just simple containers of POD 14.449 + and std::string pairs. But reference counted and the rule about not 14.450 + modifying after sharing must be followed. You'll have the urge to 14.451 + change options dynamically at some point. And you'll try to do that 14.452 + by just writing new values to the shared object. And in tests 14.453 + everything will appear to work. Then you ship and people in the 14.454 + real world start hitting read/write races in strings and then crash. 14.455 + Don't be lazy. 14.456 + 14.457 + HttpHandle. Uniquely identifies a request and can be used to 14.458 + identify it in an onCompleted() method or cancel it if it's still 14.459 + queued. But as soon as a request's onCompleted() invocation 14.460 + returns, the handle becomes invalid and may be reused immediately 14.461 + for new requests. Don't hold on to handles after notification. 14.462 + 14.463 + 14.464 +5. And Still More Refinements 14.465 + 14.466 + (Note: The following refinements are just code fragments. They 14.467 + don't directly fit into the working example above. But they 14.468 + demonstrate several idioms you'll want to copy.) 14.469 + 14.470 + LLSD, std::streambuf, std::iostream. The read(), write() and 14.471 + append() methods may be adequate for your purposes. But we use a 14.472 + lot of LLSD. Its interfaces aren't particularly compatible with 14.473 + BufferArray. And so two adapters are available to give 14.474 + stream-like behaviors: BufferArrayStreamBuf and BufferArrayStream, 14.475 + which implement the std::streambuf and std::iostream interfaces, 14.476 + respectively. 14.477 + 14.478 + A std::streambuf interface isn't something you'll want to use 14.479 + directly. Instead, you'll use the much friendlier std::iostream 14.480 + interface found in BufferArrayStream. This adapter gives you all 14.481 + the '>>' and '<<' operators you'll want as well as working 14.482 + directly with the LLSD conversion operators. 14.483 + 14.484 + Some new headers: 14.485 + 14.486 + 14.487 + #include "bufferstream.h" 14.488 + #include "llsdserialize.h" 14.489 + 14.490 + 14.491 + And an updated fragment based on onCompleted() above: 14.492 + 14.493 + 14.494 + // Successful request. Try to fetch the data 14.495 + LLCore::BufferArray * data = response->getBody(); 14.496 + LLSD resp_llsd; 14.497 + 14.498 + if (data && data->size()) 14.499 + { 14.500 + // There's some data and we expect this to be 14.501 + // LLSD. Checking of content type and validation 14.502 + // during parsing would be admirable additions. 14.503 + // But we'll forgo that now. 14.504 + LLCore::BufferArrayStream data_stream(data); 14.505 + LLSDSerialize::fromXML(resp_llsd, data_stream); 14.506 + } 14.507 + LL_INFOS("Hack") << "LLSD Received: " << resp_llsd << LL_ENDL; 14.508 + } 14.509 + else 14.510 + { 14.511 + 14.512 + 14.513 + Converting an LLSD object into an XML stream stored in a 14.514 + BufferArray is just the reverse of the above: 14.515 + 14.516 + 14.517 + BufferArray * data = new BufferArray(); 14.518 + LLCore::BufferArrayStream data_stream(data); 14.519 + 14.520 + LLSD src_llsd; 14.521 + src_llsd["foo"] = "bar"; 14.522 + 14.523 + LLSDSerialize::toXML(src_llsd, data_stream); 14.524 + 14.525 + // 'data' now contains an XML payload and can be sent 14.526 + // to a web service using the requestPut() or requestPost() 14.527 + // methods. 14.528 + ... requestPost(...); 14.529 + 14.530 + // And don't forget to release the BufferArray. 14.531 + data->release(); 14.532 + data = NULL; 14.533 + 14.534 + 14.535 + LLSD will often go hand-in-hand with BufferArray and data 14.536 + transport. But you can also do all the streaming I/O you'd expect 14.537 + of a std::iostream object: 14.538 + 14.539 + 14.540 + BufferArray * data = new BufferArray(); 14.541 + LLCore::BufferArrayStream data_stream(data); 14.542 + 14.543 + data_stream << "Hello, World!" << 29.4 << '\n'; 14.544 + std::string str; 14.545 + data_stream >> str; 14.546 + std::cout << str << std::endl; 14.547 + 14.548 + data->release(); 14.549 + // Actual delete will occur when 'data_stream' 14.550 + // falls out of scope and is destructed. 14.551 + 14.552 + 14.553 + Scoping objects and cleaning up. The examples haven't bothered 14.554 + with cleanup of objects that are no longer needed. Instead, most 14.555 + objects have been allocated as if they were global and eternal. 14.556 + You'll put the objects in more appropriate feature objects and 14.557 + clean them up as a group. Here's a checklist for actions you may 14.558 + need to take on cleanup: 14.559 + 14.560 + * Call delete on: 14.561 + o HttpHandlers created on the heap 14.562 + o HttpRequest objects 14.563 + * Call release() on: 14.564 + o BufferArray objects 14.565 + o HttpHeaders objects 14.566 + o HttpOptions objects 14.567 + o HttpResponse objects 14.568 + 14.569 + On program exit, as threads wind down, the library continues to 14.570 + operate safely. Threads don't interact via the library and even 14.571 + dangling references to HttpHandler objects are safe. If you don't 14.572 + call HttpRequest::update(), handler references are never 14.573 + dereferenced. 14.574 + 14.575 + You can take a more thorough approach to wind-down. Keep a list 14.576 + of HttpHandles (not HttpHandlers) of outstanding requests. For 14.577 + each of these, call HttpRequest::requestCancel() to cancel the 14.578 + operation. (Don't add the cancel requests' handled to the list.) 14.579 + This will cancel the outstanding requests that haven't completed. 14.580 + Canceled or completed, all requests will queue notifications. You 14.581 + can now cycle calling update() discarding responses. Continue 14.582 + until all requests notify or a few seconds have passed. 14.583 + 14.584 + Global startup and shutdown is handled in the viewer. But you can 14.585 + learn about it in the code or in the documentation in the headers. 14.586 + 14.587 + 14.588 +6. Choosing a Policy Class 14.589 + 14.590 + Now it's time to get rid of the default policy class. Take a look 14.591 + at the policy class definitions in newview/llappcorehttp.h. 14.592 + Ideally, you'll find one that's compatible with what you're doing. 14.593 + Some of the compatibility guidelines are: 14.594 + 14.595 + * Destination: Pair of host and port. Mixing requests with 14.596 + different destinations may cause more connection setup and tear 14.597 + down. 14.598 + 14.599 + * Method: http or https. Usually moot given destination. But 14.600 + mixing these may also cause connection churn. 14.601 + 14.602 + * Transfer size: If you're moving 100MB at a time and you make your 14.603 + requests to the same policy class as a lot of small, fast event 14.604 + information that fast traffic is going to get stuck behind you 14.605 + and someone's experience is going to be miserable. 14.606 + 14.607 + * Long poll requests: These are long-lived, must- do operations. 14.608 + They have a special home called AP_LONG_POLL. 14.609 + 14.610 + * Concurrency: High concurrency (5 or more) and large transfer 14.611 + sizes are incompatible. Another head-of-the-line problem. High 14.612 + concurrency is tolerated when it's desired to get maximal 14.613 + throughput. Mesh and texture downloads, for example. 14.614 + 14.615 + * Pipelined: If your requests are not idempotent, stay away from 14.616 + anything marked 'soon' or 'yes'. Hidden retries may be a 14.617 + problem for you. For now, would also recommend keeping PUT and 14.618 + POST requests out of classes that may be pipelined. Support for 14.619 + that is still a bit new. 14.620 + 14.621 + If you haven't found a compatible match, you can either create a 14.622 + new class (llappcorehttp.*) or just use AP_DEFAULT, the catchall 14.623 + class when all else fails. Inventory query operations might be a 14.624 + candidate for a new class that supported pipelining on https:. 14.625 + Same with display name lookups and other bursty-at-login 14.626 + operations. For other things, AP_DEFAULT will do what it can and 14.627 + will, in some way or another, tolerate any usage. Whether the 14.628 + users' experiences are good are for you to determine. 14.629 + 14.630 + 14.631 +7. FAQ 14.632 + 14.633 + Q1. What do these policy classes achieve? 14.634 + 14.635 + A1. Previously, HTTP-using code in the viewer was written as if 14.636 + it were some isolated, local operation that didn't have to 14.637 + consider resources, contention or impact on services and the 14.638 + larger environment. The result was an application with on the 14.639 + order of 100 HTTP launch points in its codebase that could create 14.640 + dozens or even 100's of TCP connections zeroing in on grid 14.641 + services and disrupting networking equipment, web services and 14.642 + innocent users. The use of policy classes (modeled on 14.643 + http://en.wikipedia.org/wiki/Class-based_queueing) is a means to 14.644 + restrict connection concurrency, good and necessary in itself. In 14.645 + turn, that reduces demands on an expensive resource (connection 14.646 + setup and concurrency) which relieves strain on network points. 14.647 + That enables connection keepalive and opportunites for true 14.648 + improvements in throughput and user experience. 14.649 + 14.650 + Another aspect of the classes is that they give some control over 14.651 + how competing demands for the network will be apportioned. If 14.652 + mesh fetches, texture fetches and inventory queries are all being 14.653 + made at once, the relative weights of their classes' concurrency 14.654 + limits established that apportioning. We now have an opportunity 14.655 + to balance the entire viewer system. 14.656 + 14.657 + Q2. How's that data sharing with refcounts working for you? 14.658 + 14.659 + A2. Meh. It does reduce memory churn and the frequency at which 14.660 + free blocks must be moved between threads. But it's also a design 14.661 + for static configuration and dynamic reconfiguration (not 14.662 + requiring a restart) is favored. Creating new options for every 14.663 + request isn't too bad, it a sequence of "new, fill, request, 14.664 + release" for each requested operation. That in contrast to doing 14.665 + the "new, fill, release" at startup. The bad comes in getting at 14.666 + the source data. One rule in this work was "no new thread 14.667 + problems." And one source for those is pulling setting values out 14.668 + of gSettings in threads. None of that is thread safe though we 14.669 + tend to get away with it. 14.670 + 14.671 + Q3. What needs to be done? 14.672 + 14.673 + A3. There's a To-Do list in _httpinternal.h. It has both large 14.674 + and small projects here if someone would like to try changes.
15.1 --- a/indra/llcorehttp/_httpinternal.h Mon Feb 24 11:33:41 2014 -0500 15.2 +++ b/indra/llcorehttp/_httpinternal.h Tue Feb 25 13:25:40 2014 -0500 15.3 @@ -4,7 +4,7 @@ 15.4 * 15.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 15.6 * Second Life Viewer Source Code 15.7 - * Copyright (C) 2012, Linden Research, Inc. 15.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 15.9 * 15.10 * This library is free software; you can redistribute it and/or 15.11 * modify it under the terms of the GNU Lesser General Public 15.12 @@ -36,7 +36,8 @@ 15.13 // General library to-do list 15.14 // 15.15 // - Implement policy classes. Structure is mostly there just didn't 15.16 -// need it for the first consumer. 15.17 +// need it for the first consumer. [Classes are there. More 15.18 +// advanced features, like borrowing, aren't there yet.] 15.19 // - Consider Removing 'priority' from the request interface. Its use 15.20 // in an always active class can lead to starvation of low-priority 15.21 // requests. Requires coodination of priority values across all 15.22 @@ -46,6 +47,7 @@ 15.23 // may not really need it. 15.24 // - Set/get for global policy and policy classes is clumsy. Rework 15.25 // it heading in a direction that allows for more dynamic behavior. 15.26 +// [Mostly fixed] 15.27 // - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the 15.28 // pedantic. 15.29 // - Update downloader and other long-duration services are going to 15.30 @@ -64,6 +66,12 @@ 15.31 // This won't help in the face of the router problems we've looked 15.32 // at, however. Detect starvation due to UDP activity and provide 15.33 // feedback to it. 15.34 +// - Change the transfer timeout scheme. We're less interested in 15.35 +// absolute time, in most cases, than in continuous progress. 15.36 +// - Many of the policy class settings are currently applied to the 15.37 +// entire class. Some, like connection limits, would be better 15.38 +// applied to each destination target making multiple targets 15.39 +// independent. 15.40 // 15.41 // Integration to-do list 15.42 // - LLTextureFetch still needs a major refactor. The use of 15.43 @@ -73,7 +81,6 @@ 15.44 // the main source file. 15.45 // - Expand areas of usage eventually leading to the removal of LLCurl. 15.46 // Rough order of expansion: 15.47 -// . Mesh fetch 15.48 // . Avatar names 15.49 // . Group membership lists 15.50 // . Caps access in general 15.51 @@ -97,8 +104,8 @@ 15.52 { 15.53 15.54 // Maxium number of policy classes that can be defined. 15.55 -// *TODO: Currently limited to the default class, extend. 15.56 -const int HTTP_POLICY_CLASS_LIMIT = 1; 15.57 +// *TODO: Currently limited to the default class + 1, extend. 15.58 +const int HTTP_POLICY_CLASS_LIMIT = 8; 15.59 15.60 // Debug/informational tracing. Used both 15.61 // as a global option and in per-request traces. 15.62 @@ -129,6 +136,7 @@ 15.63 // Retries and time-on-queue are not included and aren't 15.64 // accounted for. 15.65 const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; 15.66 +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L; 15.67 const long HTTP_REQUEST_TIMEOUT_MIN = 0L; 15.68 const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; 15.69 15.70 @@ -137,6 +145,11 @@ 15.71 const int HTTP_CONNECTION_LIMIT_MIN = 1; 15.72 const int HTTP_CONNECTION_LIMIT_MAX = 256; 15.73 15.74 +// Miscellaneous defaults 15.75 +const long HTTP_PIPELINING_DEFAULT = 0L; 15.76 +const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; 15.77 +const long HTTP_THROTTLE_RATE_DEFAULT = 0L; 15.78 + 15.79 // Tuning parameters 15.80 15.81 // Time worker thread sleeps after a pass through the
16.1 --- a/indra/llcorehttp/_httplibcurl.cpp Mon Feb 24 11:33:41 2014 -0500 16.2 +++ b/indra/llcorehttp/_httplibcurl.cpp Tue Feb 25 13:25:40 2014 -0500 16.3 @@ -41,7 +41,8 @@ 16.4 HttpLibcurl::HttpLibcurl(HttpService * service) 16.5 : mService(service), 16.6 mPolicyCount(0), 16.7 - mMultiHandles(NULL) 16.8 + mMultiHandles(NULL), 16.9 + mActiveHandles(NULL) 16.10 {} 16.11 16.12 16.13 @@ -77,6 +78,9 @@ 16.14 16.15 delete [] mMultiHandles; 16.16 mMultiHandles = NULL; 16.17 + 16.18 + delete [] mActiveHandles; 16.19 + mActiveHandles = NULL; 16.20 } 16.21 16.22 mPolicyCount = 0; 16.23 @@ -90,9 +94,12 @@ 16.24 16.25 mPolicyCount = policy_count; 16.26 mMultiHandles = new CURLM * [mPolicyCount]; 16.27 + mActiveHandles = new int [mPolicyCount]; 16.28 + 16.29 for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) 16.30 { 16.31 mMultiHandles[policy_class] = curl_multi_init(); 16.32 + mActiveHandles[policy_class] = 0; 16.33 } 16.34 } 16.35 16.36 @@ -110,8 +117,10 @@ 16.37 // Give libcurl some cycles to do I/O & callbacks 16.38 for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) 16.39 { 16.40 - if (! mMultiHandles[policy_class]) 16.41 + if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class]) 16.42 + { 16.43 continue; 16.44 + } 16.45 16.46 int running(0); 16.47 CURLMcode status(CURLM_CALL_MULTI_PERFORM); 16.48 @@ -132,12 +141,10 @@ 16.49 CURL * handle(msg->easy_handle); 16.50 CURLcode result(msg->data.result); 16.51 16.52 - if (completeRequest(mMultiHandles[policy_class], handle, result)) 16.53 - { 16.54 - // Request is still active, don't get too sleepy 16.55 - ret = HttpService::NORMAL; 16.56 - } 16.57 - handle = NULL; // No longer valid on return 16.58 + completeRequest(mMultiHandles[policy_class], handle, result); 16.59 + handle = NULL; // No longer valid on return 16.60 + ret = HttpService::NORMAL; // If anything completes, we may have a free slot. 16.61 + // Turning around quickly reduces connection gap by 7-10mS. 16.62 } 16.63 else if (CURLMSG_NONE == msg->msg) 16.64 { 16.65 @@ -193,6 +200,7 @@ 16.66 16.67 // On success, make operation active 16.68 mActiveOps.insert(op); 16.69 + ++mActiveHandles[op->mReqPolicy]; 16.70 } 16.71 16.72 16.73 @@ -214,6 +222,7 @@ 16.74 16.75 // Drop references 16.76 mActiveOps.erase(it); 16.77 + --mActiveHandles[op->mReqPolicy]; 16.78 op->release(); 16.79 16.80 return true; 16.81 @@ -240,7 +249,7 @@ 16.82 { 16.83 LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: " 16.84 << static_cast<HttpHandle>(op) 16.85 - << ", Status: " << op->mStatus.toHex() 16.86 + << ", Status: " << op->mStatus.toTerseString() 16.87 << LL_ENDL; 16.88 } 16.89 16.90 @@ -275,6 +284,7 @@ 16.91 16.92 // Deactivate request 16.93 mActiveOps.erase(it); 16.94 + --mActiveHandles[op->mReqPolicy]; 16.95 op->mCurlActive = false; 16.96 16.97 // Set final status of request if it hasn't failed by other mechanisms yet 16.98 @@ -316,7 +326,7 @@ 16.99 { 16.100 LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " 16.101 << static_cast<HttpHandle>(op) 16.102 - << ", Status: " << op->mStatus.toHex() 16.103 + << ", Status: " << op->mStatus.toTerseString() 16.104 << LL_ENDL; 16.105 } 16.106 16.107 @@ -336,19 +346,9 @@ 16.108 16.109 int HttpLibcurl::getActiveCountInClass(int policy_class) const 16.110 { 16.111 - int count(0); 16.112 - 16.113 - for (active_set_t::const_iterator iter(mActiveOps.begin()); 16.114 - mActiveOps.end() != iter; 16.115 - ++iter) 16.116 - { 16.117 - if ((*iter)->mReqPolicy == policy_class) 16.118 - { 16.119 - ++count; 16.120 - } 16.121 - } 16.122 - 16.123 - return count; 16.124 + llassert_always(policy_class < mPolicyCount); 16.125 + 16.126 + return mActiveHandles ? mActiveHandles[policy_class] : 0; 16.127 } 16.128 16.129
17.1 --- a/indra/llcorehttp/_httplibcurl.h Mon Feb 24 11:33:41 2014 -0500 17.2 +++ b/indra/llcorehttp/_httplibcurl.h Tue Feb 25 13:25:40 2014 -0500 17.3 @@ -4,7 +4,7 @@ 17.4 * 17.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 17.6 * Second Life Viewer Source Code 17.7 - * Copyright (C) 2012, Linden Research, Inc. 17.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 17.9 * 17.10 * This library is free software; you can redistribute it and/or 17.11 * modify it under the terms of the GNU Lesser General Public 17.12 @@ -71,16 +71,22 @@ 17.13 /// 17.14 /// @return Indication of how long this method is 17.15 /// willing to wait for next service call. 17.16 + /// 17.17 + /// Threading: called by worker thread. 17.18 HttpService::ELoopSpeed processTransport(); 17.19 17.20 /// Add request to the active list. Caller is expected to have 17.21 /// provided us with a reference count on the op to hold the 17.22 /// request. (No additional references will be added.) 17.23 + /// 17.24 + /// Threading: called by worker thread. 17.25 void addOp(HttpOpRequest * op); 17.26 17.27 /// One-time call to set the number of policy classes to be 17.28 /// serviced and to create the resources for each. Value 17.29 /// must agree with HttpPolicy::setPolicies() call. 17.30 + /// 17.31 + /// Threading: called by init thread. 17.32 void start(int policy_count); 17.33 17.34 /// Synchronously stop libcurl operations. All active requests 17.35 @@ -91,9 +97,13 @@ 17.36 /// respective reply queues. 17.37 /// 17.38 /// Can be restarted with a start() call. 17.39 + /// 17.40 + /// Threading: called by worker thread. 17.41 void shutdown(); 17.42 17.43 /// Return global and per-class counts of active requests. 17.44 + /// 17.45 + /// Threading: called by worker thread. 17.46 int getActiveCount() const; 17.47 int getActiveCountInClass(int policy_class) const; 17.48 17.49 @@ -103,6 +113,7 @@ 17.50 /// 17.51 /// @return True if handle was found and operation canceled. 17.52 /// 17.53 + /// Threading: called by worker thread. 17.54 bool cancel(HttpHandle handle); 17.55 17.56 protected: 17.57 @@ -121,7 +132,8 @@ 17.58 HttpService * mService; // Simple reference, not owner 17.59 active_set_t mActiveOps; 17.60 int mPolicyCount; 17.61 - CURLM ** mMultiHandles; 17.62 + CURLM ** mMultiHandles; // One handle per policy class 17.63 + int * mActiveHandles; // Active count per policy class 17.64 }; // end class HttpLibcurl 17.65 17.66 } // end namespace LLCore
18.1 --- a/indra/llcorehttp/_httpoperation.cpp Mon Feb 24 11:33:41 2014 -0500 18.2 +++ b/indra/llcorehttp/_httpoperation.cpp Tue Feb 25 13:25:40 2014 -0500 18.3 @@ -4,7 +4,7 @@ 18.4 * 18.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 18.6 * Second Life Viewer Source Code 18.7 - * Copyright (C) 2012, Linden Research, Inc. 18.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 18.9 * 18.10 * This library is free software; you can redistribute it and/or 18.11 * modify it under the terms of the GNU Lesser General Public 18.12 @@ -53,7 +53,7 @@ 18.13 mUserHandler(NULL), 18.14 mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), 18.15 mReqPriority(0U), 18.16 - mTracing(0) 18.17 + mTracing(HTTP_TRACE_OFF) 18.18 { 18.19 mMetricCreated = totalTime(); 18.20 } 18.21 @@ -94,7 +94,7 @@ 18.22 // Default implementation should never be called. This 18.23 // indicates an operation making a transition that isn't 18.24 // defined. 18.25 - LL_ERRS("HttpCore") << "Default stageFromRequest method may not be called." 18.26 + LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called." 18.27 << LL_ENDL; 18.28 } 18.29 18.30 @@ -104,7 +104,7 @@ 18.31 // Default implementation should never be called. This 18.32 // indicates an operation making a transition that isn't 18.33 // defined. 18.34 - LL_ERRS("HttpCore") << "Default stageFromReady method may not be called." 18.35 + LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called." 18.36 << LL_ENDL; 18.37 } 18.38 18.39 @@ -114,7 +114,7 @@ 18.40 // Default implementation should never be called. This 18.41 // indicates an operation making a transition that isn't 18.42 // defined. 18.43 - LL_ERRS("HttpCore") << "Default stageFromActive method may not be called." 18.44 + LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called." 18.45 << LL_ENDL; 18.46 } 18.47
19.1 --- a/indra/llcorehttp/_httpoperation.h Mon Feb 24 11:33:41 2014 -0500 19.2 +++ b/indra/llcorehttp/_httpoperation.h Tue Feb 25 13:25:40 2014 -0500 19.3 @@ -4,7 +4,7 @@ 19.4 * 19.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 19.6 * Second Life Viewer Source Code 19.7 - * Copyright (C) 2012, Linden Research, Inc. 19.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 19.9 * 19.10 * This library is free software; you can redistribute it and/or 19.11 * modify it under the terms of the GNU Lesser General Public 19.12 @@ -72,7 +72,7 @@ 19.13 class HttpOperation : public LLCoreInt::RefCounted 19.14 { 19.15 public: 19.16 - /// Threading: called by a consumer/application thread. 19.17 + /// Threading: called by consumer thread. 19.18 HttpOperation(); 19.19 19.20 protected: 19.21 @@ -108,7 +108,7 @@ 19.22 /// by the worker thread. This is passible data 19.23 /// until notification is performed. 19.24 /// 19.25 - /// Threading: called by application thread. 19.26 + /// Threading: called by consumer thread. 19.27 /// 19.28 void setReplyPath(HttpReplyQueue * reply_queue, 19.29 HttpHandler * handler); 19.30 @@ -141,7 +141,7 @@ 19.31 /// call to HttpRequest::update(). This method does the necessary 19.32 /// dispatching. 19.33 /// 19.34 - /// Threading: called by application thread. 19.35 + /// Threading: called by consumer thread. 19.36 /// 19.37 virtual void visitNotifier(HttpRequest *); 19.38
20.1 --- a/indra/llcorehttp/_httpoprequest.cpp Mon Feb 24 11:33:41 2014 -0500 20.2 +++ b/indra/llcorehttp/_httpoprequest.cpp Tue Feb 25 13:25:40 2014 -0500 20.3 @@ -64,6 +64,15 @@ 20.4 unsigned int * last, 20.5 unsigned int * length); 20.6 20.7 +// Similar for Retry-After headers. Only parses the delta form 20.8 +// of the header, HTTP time formats aren't interesting for client 20.9 +// purposes. 20.10 +// 20.11 +// @return 0 if successfully parsed and seconds time delta 20.12 +// returned in time argument. 20.13 +// 20.14 +int parse_retry_after_header(char * buffer, int * time); 20.15 + 20.16 20.17 // Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and 20.18 // escape and format it for a tracing line in logging. Absolutely 20.19 @@ -74,14 +83,16 @@ 20.20 std::string & safe_line); 20.21 20.22 20.23 -// OS-neutral string comparisons of various types 20.24 -int os_strncasecmp(const char *s1, const char *s2, size_t n); 20.25 -int os_strcasecmp(const char *s1, const char *s2); 20.26 -char * os_strtok_r(char *str, const char *delim, char **saveptr); 20.27 +// OS-neutral string comparisons of various types. 20.28 +int os_strcasecmp(const char * s1, const char * s2); 20.29 +char * os_strtok_r(char * str, const char * delim, char ** saveptr); 20.30 +char * os_strtrim(char * str); 20.31 +char * os_strltrim(char * str); 20.32 +void os_strlower(char * str); 20.33 20.34 - 20.35 -static const char * const hdr_whitespace(" \t"); 20.36 -static const char * const hdr_separator(": \t"); 20.37 +// Error testing and reporting for libcurl status codes 20.38 +void check_curl_easy_code(CURLcode code); 20.39 +void check_curl_easy_code(CURLcode code, int curl_setopt_option); 20.40 20.41 } // end anonymous namespace 20.42 20.43 @@ -104,12 +115,15 @@ 20.44 mCurlService(NULL), 20.45 mCurlHeaders(NULL), 20.46 mCurlBodyPos(0), 20.47 + mCurlTemp(NULL), 20.48 + mCurlTempLen(0), 20.49 mReplyBody(NULL), 20.50 mReplyOffset(0), 20.51 mReplyLength(0), 20.52 mReplyFullLength(0), 20.53 mReplyHeaders(NULL), 20.54 mPolicyRetries(0), 20.55 + mPolicy503Retries(0), 20.56 mPolicyRetryAt(HttpTime(0)), 20.57 mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) 20.58 { 20.59 @@ -153,6 +167,10 @@ 20.60 mCurlHeaders = NULL; 20.61 } 20.62 20.63 + delete [] mCurlTemp; 20.64 + mCurlTemp = NULL; 20.65 + mCurlTempLen = 0; 20.66 + 20.67 if (mReplyBody) 20.68 { 20.69 mReplyBody->release(); 20.70 @@ -208,6 +226,11 @@ 20.71 mCurlHeaders = NULL; 20.72 } 20.73 20.74 + // Also not needed on the other side 20.75 + delete [] mCurlTemp; 20.76 + mCurlTemp = NULL; 20.77 + mCurlTempLen = 0; 20.78 + 20.79 addAsReply(); 20.80 } 20.81 20.82 @@ -226,6 +249,7 @@ 20.83 response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); 20.84 } 20.85 response->setContentType(mReplyConType); 20.86 + response->setRetries(mPolicyRetries, mPolicy503Retries); 20.87 20.88 mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); 20.89 20.90 @@ -335,6 +359,10 @@ 20.91 { 20.92 mProcFlags |= PF_SAVE_HEADERS; 20.93 } 20.94 + if (options->getUseRetryAfter()) 20.95 + { 20.96 + mProcFlags |= PF_USE_RETRY_AFTER; 20.97 + } 20.98 mPolicyRetryLimit = options->getRetries(); 20.99 mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); 20.100 mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); 20.101 @@ -350,6 +378,8 @@ 20.102 // 20.103 HttpStatus HttpOpRequest::prepareRequest(HttpService * service) 20.104 { 20.105 + CURLcode code; 20.106 + 20.107 // Scrub transport and result data for retried op case 20.108 mCurlActive = false; 20.109 mCurlHandle = NULL; 20.110 @@ -379,64 +409,108 @@ 20.111 // *FIXME: better error handling later 20.112 HttpStatus status; 20.113 20.114 - // Get policy options 20.115 + // Get global policy options 20.116 HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); 20.117 20.118 mCurlHandle = LLCurl::createStandardCurlHandle(); 20.119 + if (! mCurlHandle) 20.120 + { 20.121 + // We're in trouble. We'll continue but it won't go well. 20.122 + LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing." 20.123 + << LL_ENDL; 20.124 + return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); 20.125 + } 20.126 + code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); 20.127 + check_curl_easy_code(code, CURLOPT_IPRESOLVE); 20.128 + code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); 20.129 + check_curl_easy_code(code, CURLOPT_NOSIGNAL); 20.130 + code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); 20.131 + check_curl_easy_code(code, CURLOPT_NOPROGRESS); 20.132 + code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); 20.133 + check_curl_easy_code(code, CURLOPT_URL); 20.134 + code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); 20.135 + check_curl_easy_code(code, CURLOPT_PRIVATE); 20.136 + code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); 20.137 + check_curl_easy_code(code, CURLOPT_ENCODING); 20.138 20.139 - curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); 20.140 - curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); 20.141 - curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); 20.142 - curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); 20.143 - curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); 20.144 - curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); 20.145 - curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); 20.146 + // The Linksys WRT54G V5 router has an issue with frequent 20.147 + // DNS lookups from LAN machines. If they happen too often, 20.148 + // like for every HTTP request, the router gets annoyed after 20.149 + // about 700 or so requests and starts issuing TCP RSTs to 20.150 + // new connections. Reuse the DNS lookups for even a few 20.151 + // seconds and no RSTs. 20.152 + code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); 20.153 + check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT); 20.154 + code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); 20.155 + check_curl_easy_code(code, CURLOPT_AUTOREFERER); 20.156 + code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); 20.157 + check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION); 20.158 + code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); 20.159 + check_curl_easy_code(code, CURLOPT_MAXREDIRS); 20.160 + code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); 20.161 + check_curl_easy_code(code, CURLOPT_WRITEFUNCTION); 20.162 + code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); 20.163 + check_curl_easy_code(code, CURLOPT_WRITEDATA); 20.164 + code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); 20.165 + check_curl_easy_code(code, CURLOPT_READFUNCTION); 20.166 + code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); 20.167 + check_curl_easy_code(code, CURLOPT_READDATA); 20.168 + code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); 20.169 + check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER); 20.170 + code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); 20.171 + check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); 20.172 20.173 - const std::string * opt_value(NULL); 20.174 - long opt_long(0L); 20.175 - policy.get(HttpRequest::GP_LLPROXY, &opt_long); 20.176 - if (opt_long) 20.177 + if (policy.mUseLLProxy) 20.178 { 20.179 // Use the viewer-based thread-safe API which has a 20.180 // fast/safe check for proxy enable. Would like to 20.181 // encapsulate this someway... 20.182 LLProxy::getInstance()->applyProxySettings(mCurlHandle); 20.183 } 20.184 - else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) 20.185 + else if (policy.mHttpProxy.size()) 20.186 { 20.187 // *TODO: This is fine for now but get fuller socks5/ 20.188 // authentication thing going later.... 20.189 - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); 20.190 - curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); 20.191 + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); 20.192 + check_curl_easy_code(code, CURLOPT_PROXY); 20.193 + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); 20.194 + check_curl_easy_code(code, CURLOPT_PROXYTYPE); 20.195 } 20.196 - if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) 20.197 + if (policy.mCAPath.size()) 20.198 { 20.199 - curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); 20.200 + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); 20.201 + check_curl_easy_code(code, CURLOPT_CAPATH); 20.202 } 20.203 - if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) 20.204 + if (policy.mCAFile.size()) 20.205 { 20.206 - curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); 20.207 + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); 20.208 + check_curl_easy_code(code, CURLOPT_CAINFO); 20.209 } 20.210 20.211 switch (mReqMethod) 20.212 { 20.213 case HOR_GET: 20.214 - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); 20.215 + code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); 20.216 + check_curl_easy_code(code, CURLOPT_HTTPGET); 20.217 mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); 20.218 mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); 20.219 break; 20.220 20.221 case HOR_POST: 20.222 { 20.223 - curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); 20.224 - curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); 20.225 + code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); 20.226 + check_curl_easy_code(code, CURLOPT_POST); 20.227 + code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); 20.228 + check_curl_easy_code(code, CURLOPT_ENCODING); 20.229 long data_size(0); 20.230 if (mReqBody) 20.231 { 20.232 data_size = mReqBody->size(); 20.233 } 20.234 - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); 20.235 - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); 20.236 + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); 20.237 + check_curl_easy_code(code, CURLOPT_POSTFIELDS); 20.238 + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); 20.239 + check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE); 20.240 mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); 20.241 mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); 20.242 mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); 20.243 @@ -445,14 +519,17 @@ 20.244 20.245 case HOR_PUT: 20.246 { 20.247 - curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); 20.248 + code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); 20.249 + check_curl_easy_code(code, CURLOPT_UPLOAD); 20.250 long data_size(0); 20.251 if (mReqBody) 20.252 { 20.253 data_size = mReqBody->size(); 20.254 } 20.255 - curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); 20.256 - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); 20.257 + code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); 20.258 + check_curl_easy_code(code, CURLOPT_INFILESIZE); 20.259 + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); 20.260 + check_curl_easy_code(code, CURLOPT_POSTFIELDS); 20.261 mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); 20.262 // *TODO: Should this be 'Keep-Alive' ? 20.263 mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); 20.264 @@ -470,9 +547,12 @@ 20.265 // Tracing 20.266 if (mTracing >= HTTP_TRACE_CURL_HEADERS) 20.267 { 20.268 - curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); 20.269 - curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); 20.270 - curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); 20.271 + code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); 20.272 + check_curl_easy_code(code, CURLOPT_VERBOSE); 20.273 + code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); 20.274 + check_curl_easy_code(code, CURLOPT_DEBUGDATA); 20.275 + code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); 20.276 + check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION); 20.277 } 20.278 20.279 // There's a CURLOPT for this now... 20.280 @@ -500,13 +580,22 @@ 20.281 20.282 // Request options 20.283 long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); 20.284 + long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); 20.285 if (mReqOptions) 20.286 - { 20.287 + { 20.288 timeout = mReqOptions->getTimeout(); 20.289 timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); 20.290 + xfer_timeout = mReqOptions->getTransferTimeout(); 20.291 + xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); 20.292 } 20.293 - curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); 20.294 - curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); 20.295 + if (xfer_timeout == 0L) 20.296 + { 20.297 + xfer_timeout = timeout; 20.298 + } 20.299 + code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); 20.300 + check_curl_easy_code(code, CURLOPT_TIMEOUT); 20.301 + code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); 20.302 + check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT); 20.303 20.304 // Request headers 20.305 if (mReqHeaders) 20.306 @@ -514,12 +603,15 @@ 20.307 // Caller's headers last to override 20.308 mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); 20.309 } 20.310 - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); 20.311 + code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); 20.312 + check_curl_easy_code(code, CURLOPT_HTTPHEADER); 20.313 20.314 - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) 20.315 + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) 20.316 { 20.317 - curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); 20.318 - curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); 20.319 + code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); 20.320 + check_curl_easy_code(code, CURLOPT_HEADERFUNCTION); 20.321 + code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); 20.322 + check_curl_easy_code(code, CURLOPT_HEADERDATA); 20.323 } 20.324 20.325 if (status) 20.326 @@ -560,7 +652,7 @@ 20.327 { 20.328 // Warn but continue if the read position moves beyond end-of-body 20.329 // for some reason. 20.330 - LL_WARNS("HttpCore") << "Request body position beyond body size. Truncating request body." 20.331 + LL_WARNS("CoreHttp") << "Request body position beyond body size. Truncating request body." 20.332 << LL_ENDL; 20.333 } 20.334 return 0; 20.335 @@ -577,10 +669,9 @@ 20.336 { 20.337 static const char status_line[] = "HTTP/"; 20.338 static const size_t status_line_len = sizeof(status_line) - 1; 20.339 - 20.340 - static const char con_ran_line[] = "content-range:"; 20.341 - static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; 20.342 - 20.343 + static const char con_ran_line[] = "content-range"; 20.344 + static const char con_retry_line[] = "retry-after"; 20.345 + 20.346 HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); 20.347 20.348 const size_t hdr_size(size * nmemb); 20.349 @@ -594,6 +685,7 @@ 20.350 op->mReplyOffset = 0; 20.351 op->mReplyLength = 0; 20.352 op->mReplyFullLength = 0; 20.353 + op->mReplyRetryAfter = 0; 20.354 op->mStatus = HttpStatus(); 20.355 if (op->mReplyHeaders) 20.356 { 20.357 @@ -612,6 +704,53 @@ 20.358 --wanted_hdr_size; 20.359 } 20.360 } 20.361 + 20.362 + // Copy and normalize header fragments for the following 20.363 + // stages. Would like to modify the data in-place but that 20.364 + // may not be allowed and we need one byte extra for NUL. 20.365 + // At the end of this we will have: 20.366 + // 20.367 + // If ':' present in header: 20.368 + // 1. name points to text to left of colon which 20.369 + // will be ascii lower-cased and left and right 20.370 + // trimmed of whitespace. 20.371 + // 2. value points to text to right of colon which 20.372 + // will be left trimmed of whitespace. 20.373 + // Otherwise: 20.374 + // 1. name points to header which will be left 20.375 + // trimmed of whitespace. 20.376 + // 2. value is NULL 20.377 + // Any non-NULL pointer may point to a zero-length string. 20.378 + // 20.379 + if (wanted_hdr_size >= op->mCurlTempLen) 20.380 + { 20.381 + delete [] op->mCurlTemp; 20.382 + op->mCurlTempLen = 2 * wanted_hdr_size + 1; 20.383 + op->mCurlTemp = new char [op->mCurlTempLen]; 20.384 + } 20.385 + memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size); 20.386 + op->mCurlTemp[wanted_hdr_size] = '\0'; 20.387 + char * name(op->mCurlTemp); 20.388 + char * value(strchr(name, ':')); 20.389 + if (value) 20.390 + { 20.391 + *value++ = '\0'; 20.392 + os_strlower(name); 20.393 + name = os_strtrim(name); 20.394 + value = os_strltrim(value); 20.395 + } 20.396 + else 20.397 + { 20.398 + // Doesn't look well-formed, do minimal normalization on it 20.399 + name = os_strltrim(name); 20.400 + } 20.401 + 20.402 + // Normalized, now reject headers with empty names. 20.403 + if (! *name) 20.404 + { 20.405 + // No use continuing 20.406 + return hdr_size; 20.407 + } 20.408 20.409 // Save header if caller wants them in the response 20.410 if (is_header && op->mProcFlags & PF_SAVE_HEADERS) 20.411 @@ -621,43 +760,53 @@ 20.412 { 20.413 op->mReplyHeaders = new HttpHeaders; 20.414 } 20.415 - op->mReplyHeaders->appendNormal(hdr_data, wanted_hdr_size); 20.416 + op->mReplyHeaders->append(name, value ? value : ""); 20.417 } 20.418 20.419 + // From this point, header-specific processors are free to 20.420 + // modify the header value. 20.421 + 20.422 // Detect and parse 'Content-Range' headers 20.423 - if (is_header && op->mProcFlags & PF_SCAN_RANGE_HEADER) 20.424 + if (is_header 20.425 + && op->mProcFlags & PF_SCAN_RANGE_HEADER 20.426 + && value && *value 20.427 + && ! strcmp(name, con_ran_line)) 20.428 { 20.429 - char hdr_buffer[128]; // Enough for a reasonable header 20.430 - size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); 20.431 - 20.432 - memcpy(hdr_buffer, hdr_data, frag_size); 20.433 - hdr_buffer[frag_size] = '\0'; 20.434 - if (frag_size > con_ran_line_len && 20.435 - ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) 20.436 + unsigned int first(0), last(0), length(0); 20.437 + int status; 20.438 + 20.439 + if (! (status = parse_content_range_header(value, &first, &last, &length))) 20.440 { 20.441 - unsigned int first(0), last(0), length(0); 20.442 - int status; 20.443 + // Success, record the fragment position 20.444 + op->mReplyOffset = first; 20.445 + op->mReplyLength = last - first + 1; 20.446 + op->mReplyFullLength = length; 20.447 + } 20.448 + else if (-1 == status) 20.449 + { 20.450 + // Response is badly formed and shouldn't be accepted 20.451 + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); 20.452 + } 20.453 + else 20.454 + { 20.455 + // Ignore the unparsable. 20.456 + LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" 20.457 + << std::string(hdr_data, wanted_hdr_size) 20.458 + << "'. Ignoring." 20.459 + << LL_ENDL; 20.460 + } 20.461 + } 20.462 20.463 - if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length))) 20.464 - { 20.465 - // Success, record the fragment position 20.466 - op->mReplyOffset = first; 20.467 - op->mReplyLength = last - first + 1; 20.468 - op->mReplyFullLength = length; 20.469 - } 20.470 - else if (-1 == status) 20.471 - { 20.472 - // Response is badly formed and shouldn't be accepted 20.473 - op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); 20.474 - } 20.475 - else 20.476 - { 20.477 - // Ignore the unparsable. 20.478 - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" 20.479 - << std::string(hdr_data, frag_size) 20.480 - << "'. Ignoring." 20.481 - << LL_ENDL; 20.482 - } 20.483 + // Detect and parse 'Retry-After' headers 20.484 + if (is_header 20.485 + && op->mProcFlags & PF_USE_RETRY_AFTER 20.486 + && value && *value 20.487 + && ! strcmp(name, con_retry_line)) 20.488 + { 20.489 + int time(0); 20.490 + if (! parse_retry_after_header(value, &time)) 20.491 + { 20.492 + op->mReplyRetryAfter = time; 20.493 } 20.494 } 20.495 20.496 @@ -772,14 +921,16 @@ 20.497 unsigned int * last, 20.498 unsigned int * length) 20.499 { 20.500 + static const char * const hdr_whitespace(" \t"); 20.501 + 20.502 char * tok_state(NULL), * tok(NULL); 20.503 bool match(true); 20.504 20.505 - if (! os_strtok_r(buffer, hdr_separator, &tok_state)) 20.506 + if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state))) 20.507 match = false; 20.508 - if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) 20.509 - match = 0 == os_strcasecmp("bytes", tok); 20.510 - if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) 20.511 + else 20.512 + match = (0 == os_strcasecmp("bytes", tok)); 20.513 + if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) 20.514 match = false; 20.515 if (match) 20.516 { 20.517 @@ -818,6 +969,25 @@ 20.518 } 20.519 20.520 20.521 +int parse_retry_after_header(char * buffer, int * time) 20.522 +{ 20.523 + char * endptr(buffer); 20.524 + long lcl_time(strtol(buffer, &endptr, 10)); 20.525 + if (*endptr == '\0' && endptr != buffer && lcl_time > 0) 20.526 + { 20.527 + *time = lcl_time; 20.528 + return 0; 20.529 + } 20.530 + 20.531 + // Could attempt to parse HTTP time here but we're not really 20.532 + // interested in it. Scheduling based on wallclock time on 20.533 + // user hardware will lead to tears. 20.534 + 20.535 + // Header is there but badly/unexpectedly formed, try to ignore it. 20.536 + return 1; 20.537 +} 20.538 + 20.539 + 20.540 void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) 20.541 { 20.542 std::string out; 20.543 @@ -854,15 +1024,6 @@ 20.544 } 20.545 20.546 20.547 -int os_strncasecmp(const char *s1, const char *s2, size_t n) 20.548 -{ 20.549 -#if LL_WINDOWS 20.550 - return _strnicmp(s1, s2, n); 20.551 -#else 20.552 - return strncasecmp(s1, s2, n); 20.553 -#endif // LL_WINDOWS 20.554 -} 20.555 - 20.556 20.557 int os_strcasecmp(const char *s1, const char *s2) 20.558 { 20.559 @@ -884,6 +1045,73 @@ 20.560 } 20.561 20.562 20.563 +void os_strlower(char * str) 20.564 +{ 20.565 + for (char c(0); (c = *str); ++str) 20.566 + { 20.567 + *str = tolower(c); 20.568 + } 20.569 +} 20.570 + 20.571 + 20.572 +char * os_strtrim(char * lstr) 20.573 +{ 20.574 + while (' ' == *lstr || '\t' == *lstr) 20.575 + { 20.576 + ++lstr; 20.577 + } 20.578 + if (*lstr) 20.579 + { 20.580 + char * rstr(lstr + strlen(lstr)); 20.581 + while (lstr < rstr && *--rstr) 20.582 + { 20.583 + if (' ' == *rstr || '\t' == *rstr) 20.584 + { 20.585 + *rstr = '\0'; 20.586 + } 20.587 + } 20.588 + llassert(lstr <= rstr); 20.589 + } 20.590 + return lstr; 20.591 +} 20.592 + 20.593 + 20.594 +char * os_strltrim(char * lstr) 20.595 +{ 20.596 + while (' ' == *lstr || '\t' == *lstr) 20.597 + { 20.598 + ++lstr; 20.599 + } 20.600 + return lstr; 20.601 +} 20.602 + 20.603 + 20.604 +void check_curl_easy_code(CURLcode code, int curl_setopt_option) 20.605 +{ 20.606 + if (CURLE_OK != code) 20.607 + { 20.608 + // Comment from old llcurl code which may no longer apply: 20.609 + // 20.610 + // linux appears to throw a curl error once per session for a bad initialization 20.611 + // at a pretty random time (when enabling cookies). 20.612 + LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) 20.613 + << ", curl_easy_setopt option: " << curl_setopt_option 20.614 + << LL_ENDL; 20.615 + } 20.616 +} 20.617 + 20.618 + 20.619 +void check_curl_easy_code(CURLcode code) 20.620 +{ 20.621 + if (CURLE_OK != code) 20.622 + { 20.623 + // Comment from old llcurl code which may no longer apply: 20.624 + // 20.625 + // linux appears to throw a curl error once per session for a bad initialization 20.626 + // at a pretty random time (when enabling cookies). 20.627 + LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) 20.628 + << LL_ENDL; 20.629 + } 20.630 +} 20.631 + 20.632 } // end anonymous namespace 20.633 - 20.634 -
21.1 --- a/indra/llcorehttp/_httpoprequest.h Mon Feb 24 11:33:41 2014 -0500 21.2 +++ b/indra/llcorehttp/_httpoprequest.h Tue Feb 25 13:25:40 2014 -0500 21.3 @@ -4,7 +4,7 @@ 21.4 * 21.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 21.6 * Second Life Viewer Source Code 21.7 - * Copyright (C) 2012, Linden Research, Inc. 21.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 21.9 * 21.10 * This library is free software; you can redistribute it and/or 21.11 * modify it under the terms of the GNU Lesser General Public 21.12 @@ -157,6 +157,7 @@ 21.13 unsigned int mProcFlags; 21.14 static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; 21.15 static const unsigned int PF_SAVE_HEADERS = 0x00000002U; 21.16 + static const unsigned int PF_USE_RETRY_AFTER = 0x00000004U; 21.17 21.18 public: 21.19 // Request data 21.20 @@ -174,6 +175,8 @@ 21.21 HttpService * mCurlService; 21.22 curl_slist * mCurlHeaders; 21.23 size_t mCurlBodyPos; 21.24 + char * mCurlTemp; // Scratch buffer for header processing 21.25 + size_t mCurlTempLen; 21.26 21.27 // Result data 21.28 HttpStatus mStatus; 21.29 @@ -183,9 +186,11 @@ 21.30 size_t mReplyFullLength; 21.31 HttpHeaders * mReplyHeaders; 21.32 std::string mReplyConType; 21.33 + int mReplyRetryAfter; 21.34 21.35 // Policy data 21.36 int mPolicyRetries; 21.37 + int mPolicy503Retries; 21.38 HttpTime mPolicyRetryAt; 21.39 int mPolicyRetryLimit; 21.40 }; // end class HttpOpRequest
22.1 --- a/indra/llcorehttp/_httpopsetget.cpp Mon Feb 24 11:33:41 2014 -0500 22.2 +++ b/indra/llcorehttp/_httpopsetget.cpp Tue Feb 25 13:25:40 2014 -0500 22.3 @@ -4,7 +4,7 @@ 22.4 * 22.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 22.6 * Second Life Viewer Source Code 22.7 - * Copyright (C) 2012, Linden Research, Inc. 22.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 22.9 * 22.10 * This library is free software; you can redistribute it and/or 22.11 * modify it under the terms of the GNU Lesser General Public 22.12 @@ -27,6 +27,7 @@ 22.13 #include "_httpopsetget.h" 22.14 22.15 #include "httpcommon.h" 22.16 +#include "httprequest.h" 22.17 22.18 #include "_httpservice.h" 22.19 #include "_httppolicy.h" 22.20 @@ -43,10 +44,11 @@ 22.21 22.22 HttpOpSetGet::HttpOpSetGet() 22.23 : HttpOperation(), 22.24 - mIsGlobal(false), 22.25 - mDoSet(false), 22.26 - mSetting(-1), // Nothing requested 22.27 - mLongValue(0L) 22.28 + mReqOption(HttpRequest::PO_CONNECTION_LIMIT), 22.29 + mReqClass(HttpRequest::INVALID_POLICY_ID), 22.30 + mReqDoSet(false), 22.31 + mReqLongValue(0L), 22.32 + mReplyLongValue(0L) 22.33 {} 22.34 22.35 22.36 @@ -54,37 +56,84 @@ 22.37 {} 22.38 22.39 22.40 -void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting) 22.41 +HttpStatus HttpOpSetGet::setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass) 22.42 { 22.43 - mIsGlobal = true; 22.44 - mSetting = setting; 22.45 + HttpStatus status; 22.46 + 22.47 + mReqOption = opt; 22.48 + mReqClass = pclass; 22.49 + return status; 22.50 } 22.51 22.52 22.53 -void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value) 22.54 +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value) 22.55 { 22.56 - mIsGlobal = true; 22.57 - mDoSet = true; 22.58 - mSetting = setting; 22.59 - mStrValue = value; 22.60 + HttpStatus status; 22.61 + 22.62 + if (! HttpService::sOptionDesc[opt].mIsLong) 22.63 + { 22.64 + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); 22.65 + } 22.66 + if (! HttpService::sOptionDesc[opt].mIsDynamic) 22.67 + { 22.68 + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); 22.69 + } 22.70 + 22.71 + mReqOption = opt; 22.72 + mReqClass = pclass; 22.73 + mReqDoSet = true; 22.74 + mReqLongValue = value; 22.75 + 22.76 + return status; 22.77 +} 22.78 + 22.79 + 22.80 +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value) 22.81 +{ 22.82 + HttpStatus status; 22.83 + 22.84 + if (HttpService::sOptionDesc[opt].mIsLong) 22.85 + { 22.86 + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); 22.87 + } 22.88 + if (! HttpService::sOptionDesc[opt].mIsDynamic) 22.89 + { 22.90 + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); 22.91 + } 22.92 + 22.93 + mReqOption = opt; 22.94 + mReqClass = pclass; 22.95 + mReqDoSet = true; 22.96 + mReqStrValue = value; 22.97 + 22.98 + return status; 22.99 } 22.100 22.101 22.102 void HttpOpSetGet::stageFromRequest(HttpService * service) 22.103 { 22.104 - HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions()); 22.105 - HttpRequest::EGlobalPolicy setting(static_cast<HttpRequest::EGlobalPolicy>(mSetting)); 22.106 - 22.107 - if (mDoSet) 22.108 + if (mReqDoSet) 22.109 { 22.110 - mStatus = pol_opt.set(setting, mStrValue); 22.111 + if (HttpService::sOptionDesc[mReqOption].mIsLong) 22.112 + { 22.113 + mStatus = service->setPolicyOption(mReqOption, mReqClass, 22.114 + mReqLongValue, &mReplyLongValue); 22.115 + } 22.116 + else 22.117 + { 22.118 + mStatus = service->setPolicyOption(mReqOption, mReqClass, 22.119 + mReqStrValue, &mReplyStrValue); 22.120 + } 22.121 } 22.122 - if (mStatus) 22.123 + else 22.124 { 22.125 - const std::string * value(NULL); 22.126 - if ((mStatus = pol_opt.get(setting, &value))) 22.127 + if (HttpService::sOptionDesc[mReqOption].mIsLong) 22.128 { 22.129 - mStrValue = *value; 22.130 + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyLongValue); 22.131 + } 22.132 + else 22.133 + { 22.134 + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyStrValue); 22.135 } 22.136 } 22.137
23.1 --- a/indra/llcorehttp/_httpopsetget.h Mon Feb 24 11:33:41 2014 -0500 23.2 +++ b/indra/llcorehttp/_httpopsetget.h Tue Feb 25 13:25:40 2014 -0500 23.3 @@ -4,7 +4,7 @@ 23.4 * 23.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 23.6 * Second Life Viewer Source Code 23.7 - * Copyright (C) 2012, Linden Research, Inc. 23.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 23.9 * 23.10 * This library is free software; you can redistribute it and/or 23.11 * modify it under the terms of the GNU Lesser General Public 23.12 @@ -46,7 +46,10 @@ 23.13 /// configuration settings. 23.14 /// 23.15 /// *NOTE: Expect this to change. Don't really like it yet. 23.16 - 23.17 +/// 23.18 +/// *TODO: Can't return values to caller yet. Need to do 23.19 +/// something better with HttpResponse and visitNotifier(). 23.20 +/// 23.21 class HttpOpSetGet : public HttpOperation 23.22 { 23.23 public: 23.24 @@ -61,19 +64,23 @@ 23.25 23.26 public: 23.27 /// Threading: called by application thread 23.28 - void setupGet(HttpRequest::EGlobalPolicy setting); 23.29 - void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); 23.30 + HttpStatus setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass); 23.31 + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value); 23.32 + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value); 23.33 23.34 virtual void stageFromRequest(HttpService *); 23.35 23.36 public: 23.37 // Request data 23.38 - bool mIsGlobal; 23.39 - bool mDoSet; 23.40 - int mSetting; 23.41 - long mLongValue; 23.42 - std::string mStrValue; 23.43 + HttpRequest::EPolicyOption mReqOption; 23.44 + HttpRequest::policy_t mReqClass; 23.45 + bool mReqDoSet; 23.46 + long mReqLongValue; 23.47 + std::string mReqStrValue; 23.48 23.49 + // Reply Data 23.50 + long mReplyLongValue; 23.51 + std::string mReplyStrValue; 23.52 }; // end class HttpOpSetGet 23.53 23.54
24.1 --- a/indra/llcorehttp/_httppolicy.cpp Mon Feb 24 11:33:41 2014 -0500 24.2 +++ b/indra/llcorehttp/_httppolicy.cpp Tue Feb 25 13:25:40 2014 -0500 24.3 @@ -41,57 +41,70 @@ 24.4 24.5 24.6 // Per-policy-class data for a running system. 24.7 -// Collection of queues, parameters, history, metrics, etc. 24.8 +// Collection of queues, options and other data 24.9 // for a single policy class. 24.10 // 24.11 // Threading: accessed only by worker thread 24.12 -struct HttpPolicy::State 24.13 +struct HttpPolicy::ClassState 24.14 { 24.15 public: 24.16 - State() 24.17 - : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), 24.18 - mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), 24.19 - mConnMin(1), 24.20 - mNextSample(0), 24.21 - mErrorCount(0), 24.22 - mErrorFactor(0) 24.23 + ClassState() 24.24 + : mThrottleEnd(0), 24.25 + mThrottleLeft(0L), 24.26 + mRequestCount(0L) 24.27 {} 24.28 24.29 HttpReadyQueue mReadyQueue; 24.30 HttpRetryQueue mRetryQueue; 24.31 24.32 HttpPolicyClass mOptions; 24.33 - 24.34 - long mConnMax; 24.35 - long mConnAt; 24.36 - long mConnMin; 24.37 - 24.38 - HttpTime mNextSample; 24.39 - unsigned long mErrorCount; 24.40 - unsigned long mErrorFactor; 24.41 + HttpTime mThrottleEnd; 24.42 + long mThrottleLeft; 24.43 + long mRequestCount; 24.44 }; 24.45 24.46 24.47 HttpPolicy::HttpPolicy(HttpService * service) 24.48 - : mActiveClasses(0), 24.49 - mState(NULL), 24.50 - mService(service) 24.51 -{} 24.52 + : mService(service) 24.53 +{ 24.54 + // Create default class 24.55 + mClasses.push_back(new ClassState()); 24.56 +} 24.57 24.58 24.59 HttpPolicy::~HttpPolicy() 24.60 { 24.61 shutdown(); 24.62 + 24.63 + for (class_list_t::iterator it(mClasses.begin()); it != mClasses.end(); ++it) 24.64 + { 24.65 + delete (*it); 24.66 + } 24.67 + mClasses.clear(); 24.68 24.69 mService = NULL; 24.70 } 24.71 24.72 24.73 +HttpRequest::policy_t HttpPolicy::createPolicyClass() 24.74 +{ 24.75 + const HttpRequest::policy_t policy_class(mClasses.size()); 24.76 + if (policy_class >= HTTP_POLICY_CLASS_LIMIT) 24.77 + { 24.78 + return HttpRequest::INVALID_POLICY_ID; 24.79 + } 24.80 + mClasses.push_back(new ClassState()); 24.81 + return policy_class; 24.82 +} 24.83 + 24.84 + 24.85 void HttpPolicy::shutdown() 24.86 { 24.87 - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) 24.88 + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) 24.89 { 24.90 - HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); 24.91 + ClassState & state(*mClasses[policy_class]); 24.92 + 24.93 + HttpRetryQueue & retryq(state.mRetryQueue); 24.94 while (! retryq.empty()) 24.95 { 24.96 HttpOpRequest * op(retryq.top()); 24.97 @@ -101,7 +114,7 @@ 24.98 op->release(); 24.99 } 24.100 24.101 - HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); 24.102 + HttpReadyQueue & readyq(state.mReadyQueue); 24.103 while (! readyq.empty()) 24.104 { 24.105 HttpOpRequest * op(readyq.top()); 24.106 @@ -111,28 +124,11 @@ 24.107 op->release(); 24.108 } 24.109 } 24.110 - delete [] mState; 24.111 - mState = NULL; 24.112 - mActiveClasses = 0; 24.113 } 24.114 24.115 24.116 -void HttpPolicy::start(const HttpPolicyGlobal & global, 24.117 - const std::vector<HttpPolicyClass> & classes) 24.118 -{ 24.119 - llassert_always(! mState); 24.120 - 24.121 - mGlobalOptions = global; 24.122 - mActiveClasses = classes.size(); 24.123 - mState = new State [mActiveClasses]; 24.124 - for (int i(0); i < mActiveClasses; ++i) 24.125 - { 24.126 - mState[i].mOptions = classes[i]; 24.127 - mState[i].mConnMax = classes[i].mConnectionLimit; 24.128 - mState[i].mConnAt = mState[i].mConnMax; 24.129 - mState[i].mConnMin = 2; 24.130 - } 24.131 -} 24.132 +void HttpPolicy::start() 24.133 +{} 24.134 24.135 24.136 void HttpPolicy::addOp(HttpOpRequest * op) 24.137 @@ -140,7 +136,8 @@ 24.138 const int policy_class(op->mReqPolicy); 24.139 24.140 op->mPolicyRetries = 0; 24.141 - mState[policy_class].mReadyQueue.push(op); 24.142 + op->mPolicy503Retries = 0; 24.143 + mClasses[policy_class]->mReadyQueue.push(op); 24.144 } 24.145 24.146 24.147 @@ -155,25 +152,39 @@ 24.148 5000000 // ... to every 5.0 S. 24.149 }; 24.150 static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); 24.151 - 24.152 + static const HttpStatus error_503(503); 24.153 + 24.154 const HttpTime now(totalTime()); 24.155 const int policy_class(op->mReqPolicy); 24.156 - 24.157 - const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); 24.158 + HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); 24.159 + bool external_delta(false); 24.160 + 24.161 + if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30) 24.162 + { 24.163 + delta = op->mReplyRetryAfter * U64L(1000000); 24.164 + external_delta = true; 24.165 + } 24.166 op->mPolicyRetryAt = now + delta; 24.167 ++op->mPolicyRetries; 24.168 - LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) 24.169 - << " retry " << op->mPolicyRetries 24.170 - << " scheduled for +" << (delta / HttpTime(1000)) 24.171 - << " mS. Status: " << op->mStatus.toHex() 24.172 - << LL_ENDL; 24.173 - if (op->mTracing > 0) 24.174 + if (error_503 == op->mStatus) 24.175 + { 24.176 + ++op->mPolicy503Retries; 24.177 + } 24.178 + LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) 24.179 + << " retry " << op->mPolicyRetries 24.180 + << " scheduled in " << (delta / HttpTime(1000)) 24.181 + << " mS (" << (external_delta ? "external" : "internal") 24.182 + << "). Status: " << op->mStatus.toTerseString() 24.183 + << LL_ENDL; 24.184 + if (op->mTracing > HTTP_TRACE_OFF) 24.185 { 24.186 LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " 24.187 << static_cast<HttpHandle>(op) 24.188 + << ", Delta: " << (delta / HttpTime(1000)) 24.189 + << ", Retries: " << op->mPolicyRetries 24.190 << LL_ENDL; 24.191 } 24.192 - mState[policy_class].mRetryQueue.push(op); 24.193 + mClasses[policy_class]->mRetryQueue.push(op); 24.194 } 24.195 24.196 24.197 @@ -188,21 +199,43 @@ 24.198 // the worker thread may sleep hard otherwise will ask for 24.199 // normal polling frequency. 24.200 // 24.201 +// Implements a client-side request rate throttle as well. 24.202 +// This is intended to mimic and predict throttling behavior 24.203 +// of grid services but that is difficult to do with different 24.204 +// time bases. This also represents a rigid coupling between 24.205 +// viewer and server that makes it hard to change parameters 24.206 +// and I hope we can make this go away with pipelining. 24.207 +// 24.208 HttpService::ELoopSpeed HttpPolicy::processReadyQueue() 24.209 { 24.210 const HttpTime now(totalTime()); 24.211 HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); 24.212 HttpLibcurl & transport(mService->getTransport()); 24.213 24.214 - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) 24.215 + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) 24.216 { 24.217 - State & state(mState[policy_class]); 24.218 - int active(transport.getActiveCountInClass(policy_class)); 24.219 - int needed(state.mConnAt - active); // Expect negatives here 24.220 - 24.221 + ClassState & state(*mClasses[policy_class]); 24.222 HttpRetryQueue & retryq(state.mRetryQueue); 24.223 HttpReadyQueue & readyq(state.mReadyQueue); 24.224 + 24.225 + if (retryq.empty() && readyq.empty()) 24.226 + { 24.227 + continue; 24.228 + } 24.229 24.230 + const bool throttle_enabled(state.mOptions.mThrottleRate > 0L); 24.231 + const bool throttle_current(throttle_enabled && now < state.mThrottleEnd); 24.232 + 24.233 + if (throttle_current && state.mThrottleLeft <= 0) 24.234 + { 24.235 + // Throttled condition, don't serve this class but don't sleep hard. 24.236 + result = HttpService::NORMAL; 24.237 + continue; 24.238 + } 24.239 + 24.240 + int active(transport.getActiveCountInClass(policy_class)); 24.241 + int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here 24.242 + 24.243 if (needed > 0) 24.244 { 24.245 // First see if we have any retries... 24.246 @@ -216,10 +249,27 @@ 24.247 24.248 op->stageFromReady(mService); 24.249 op->release(); 24.250 - 24.251 + 24.252 + ++state.mRequestCount; 24.253 --needed; 24.254 + if (throttle_enabled) 24.255 + { 24.256 + if (now >= state.mThrottleEnd) 24.257 + { 24.258 + // Throttle expired, move to next window 24.259 + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft 24.260 + << " requests to go and " << state.mRequestCount 24.261 + << " requests issued." << LL_ENDL; 24.262 + state.mThrottleLeft = state.mOptions.mThrottleRate; 24.263 + state.mThrottleEnd = now + HttpTime(1000000); 24.264 + } 24.265 + if (--state.mThrottleLeft <= 0) 24.266 + { 24.267 + goto throttle_on; 24.268 + } 24.269 + } 24.270 } 24.271 - 24.272 + 24.273 // Now go on to the new requests... 24.274 while (needed > 0 && ! readyq.empty()) 24.275 { 24.276 @@ -229,10 +279,29 @@ 24.277 op->stageFromReady(mService); 24.278 op->release(); 24.279 24.280 + ++state.mRequestCount; 24.281 --needed; 24.282 + if (throttle_enabled) 24.283 + { 24.284 + if (now >= state.mThrottleEnd) 24.285 + { 24.286 + // Throttle expired, move to next window 24.287 + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft 24.288 + << " requests to go and " << state.mRequestCount 24.289 + << " requests issued." << LL_ENDL; 24.290 + state.mThrottleLeft = state.mOptions.mThrottleRate; 24.291 + state.mThrottleEnd = now + HttpTime(1000000); 24.292 + } 24.293 + if (--state.mThrottleLeft <= 0) 24.294 + { 24.295 + goto throttle_on; 24.296 + } 24.297 + } 24.298 } 24.299 } 24.300 - 24.301 + 24.302 + throttle_on: 24.303 + 24.304 if (! readyq.empty() || ! retryq.empty()) 24.305 { 24.306 // If anything is ready, continue looping... 24.307 @@ -246,9 +315,9 @@ 24.308 24.309 bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) 24.310 { 24.311 - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) 24.312 + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) 24.313 { 24.314 - State & state(mState[policy_class]); 24.315 + ClassState & state(*mClasses[policy_class]); 24.316 // We don't scan retry queue because a priority change there 24.317 // is meaningless. The request will be issued based on retry 24.318 // intervals not priority value, which is now moot. 24.319 @@ -276,9 +345,9 @@ 24.320 24.321 bool HttpPolicy::cancel(HttpHandle handle) 24.322 { 24.323 - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) 24.324 + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) 24.325 { 24.326 - State & state(mState[policy_class]); 24.327 + ClassState & state(*mClasses[policy_class]); 24.328 24.329 // Scan retry queue 24.330 HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); 24.331 @@ -337,14 +406,14 @@ 24.332 LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) 24.333 << " failed after " << op->mPolicyRetries 24.334 << " retries. Reason: " << op->mStatus.toString() 24.335 - << " (" << op->mStatus.toHex() << ")" 24.336 + << " (" << op->mStatus.toTerseString() << ")" 24.337 << LL_ENDL; 24.338 } 24.339 else if (op->mPolicyRetries) 24.340 { 24.341 - LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) 24.342 - << " succeeded on retry " << op->mPolicyRetries << "." 24.343 - << LL_ENDL; 24.344 + LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) 24.345 + << " succeeded on retry " << op->mPolicyRetries << "." 24.346 + << LL_ENDL; 24.347 } 24.348 24.349 op->stageFromActive(mService); 24.350 @@ -352,13 +421,21 @@ 24.351 return false; // not active 24.352 } 24.353 24.354 + 24.355 +HttpPolicyClass & HttpPolicy::getClassOptions(HttpRequest::policy_t pclass) 24.356 +{ 24.357 + llassert_always(pclass >= 0 && pclass < mClasses.size()); 24.358 + 24.359 + return mClasses[pclass]->mOptions; 24.360 +} 24.361 + 24.362 24.363 int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const 24.364 { 24.365 - if (policy_class < mActiveClasses) 24.366 + if (policy_class < mClasses.size()) 24.367 { 24.368 - return (mState[policy_class].mReadyQueue.size() 24.369 - + mState[policy_class].mRetryQueue.size()); 24.370 + return (mClasses[policy_class]->mReadyQueue.size() 24.371 + + mClasses[policy_class]->mRetryQueue.size()); 24.372 } 24.373 return 0; 24.374 }
25.1 --- a/indra/llcorehttp/_httppolicy.h Mon Feb 24 11:33:41 2014 -0500 25.2 +++ b/indra/llcorehttp/_httppolicy.h Tue Feb 25 13:25:40 2014 -0500 25.3 @@ -4,7 +4,7 @@ 25.4 * 25.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 25.6 * Second Life Viewer Source Code 25.7 - * Copyright (C) 2012, Linden Research, Inc. 25.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 25.9 * 25.10 * This library is free software; you can redistribute it and/or 25.11 * modify it under the terms of the GNU Lesser General Public 25.12 @@ -60,6 +60,9 @@ 25.13 void operator=(const HttpPolicy &); // Not defined 25.14 25.15 public: 25.16 + /// Threading: called by init thread. 25.17 + HttpRequest::policy_t createPolicyClass(); 25.18 + 25.19 /// Cancel all ready and retry requests sending them to 25.20 /// their notification queues. Release state resources 25.21 /// making further request handling impossible. 25.22 @@ -71,9 +74,8 @@ 25.23 /// requests. One-time call invoked before starting 25.24 /// the worker thread. 25.25 /// 25.26 - /// Threading: called by application thread 25.27 - void start(const HttpPolicyGlobal & global, 25.28 - const std::vector<HttpPolicyClass> & classes); 25.29 + /// Threading: called by init thread 25.30 + void start(); 25.31 25.32 /// Give the policy layer some cycles to scan the ready 25.33 /// queue promoting higher-priority requests to active 25.34 @@ -93,7 +95,7 @@ 25.35 /// and should not be modified by anyone until retrieved 25.36 /// from queue. 25.37 /// 25.38 - /// Threading: called by any thread 25.39 + /// Threading: called by worker thread 25.40 void addOp(HttpOpRequest *); 25.41 25.42 /// Similar to addOp, used when a caller wants to retry a 25.43 @@ -130,30 +132,39 @@ 25.44 /// Threading: called by worker thread 25.45 bool stageAfterCompletion(HttpOpRequest * op); 25.46 25.47 - // Get pointer to global policy options. Caller is expected 25.48 - // to do context checks like no setting once running. 25.49 + /// Get a reference to global policy options. Caller is expected 25.50 + /// to do context checks like no setting once running. These 25.51 + /// are done, for example, in @see HttpService interfaces. 25.52 /// 25.53 /// Threading: called by any thread *but* the object may 25.54 /// only be modified by the worker thread once running. 25.55 - /// 25.56 HttpPolicyGlobal & getGlobalOptions() 25.57 { 25.58 return mGlobalOptions; 25.59 } 25.60 25.61 + /// Get a reference to class policy options. Caller is expected 25.62 + /// to do context checks like no setting once running. These 25.63 + /// are done, for example, in @see HttpService interfaces. 25.64 + /// 25.65 + /// Threading: called by any thread *but* the object may 25.66 + /// only be modified by the worker thread once running and 25.67 + /// read accesses by other threads are exposed to races at 25.68 + /// that point. 25.69 + HttpPolicyClass & getClassOptions(HttpRequest::policy_t pclass); 25.70 + 25.71 /// Get ready counts for a particular policy class 25.72 /// 25.73 /// Threading: called by worker thread 25.74 int getReadyCount(HttpRequest::policy_t policy_class) const; 25.75 25.76 protected: 25.77 - struct State; 25.78 - 25.79 - int mActiveClasses; 25.80 - State * mState; 25.81 + struct ClassState; 25.82 + typedef std::vector<ClassState *> class_list_t; 25.83 + 25.84 + HttpPolicyGlobal mGlobalOptions; 25.85 + class_list_t mClasses; 25.86 HttpService * mService; // Naked pointer, not refcounted, not owner 25.87 - HttpPolicyGlobal mGlobalOptions; 25.88 - 25.89 }; // end class HttpPolicy 25.90 25.91 } // end namespace LLCore
26.1 --- a/indra/llcorehttp/_httppolicyclass.cpp Mon Feb 24 11:33:41 2014 -0500 26.2 +++ b/indra/llcorehttp/_httppolicyclass.cpp Tue Feb 25 13:25:40 2014 -0500 26.3 @@ -4,7 +4,7 @@ 26.4 * 26.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 26.6 * Second Life Viewer Source Code 26.7 - * Copyright (C) 2012, Linden Research, Inc. 26.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 26.9 * 26.10 * This library is free software; you can redistribute it and/or 26.11 * modify it under the terms of the GNU Lesser General Public 26.12 @@ -34,10 +34,10 @@ 26.13 26.14 26.15 HttpPolicyClass::HttpPolicyClass() 26.16 - : mSetMask(0UL), 26.17 - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), 26.18 + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), 26.19 mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), 26.20 - mPipelining(0) 26.21 + mPipelining(HTTP_PIPELINING_DEFAULT), 26.22 + mThrottleRate(HTTP_THROTTLE_RATE_DEFAULT) 26.23 {} 26.24 26.25 26.26 @@ -49,75 +49,75 @@ 26.27 { 26.28 if (this != &other) 26.29 { 26.30 - mSetMask = other.mSetMask; 26.31 mConnectionLimit = other.mConnectionLimit; 26.32 mPerHostConnectionLimit = other.mPerHostConnectionLimit; 26.33 mPipelining = other.mPipelining; 26.34 + mThrottleRate = other.mThrottleRate; 26.35 } 26.36 return *this; 26.37 } 26.38 26.39 26.40 HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) 26.41 - : mSetMask(other.mSetMask), 26.42 - mConnectionLimit(other.mConnectionLimit), 26.43 + : mConnectionLimit(other.mConnectionLimit), 26.44 mPerHostConnectionLimit(other.mPerHostConnectionLimit), 26.45 - mPipelining(other.mPipelining) 26.46 + mPipelining(other.mPipelining), 26.47 + mThrottleRate(other.mThrottleRate) 26.48 {} 26.49 26.50 26.51 -HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) 26.52 +HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) 26.53 { 26.54 switch (opt) 26.55 { 26.56 - case HttpRequest::CP_CONNECTION_LIMIT: 26.57 + case HttpRequest::PO_CONNECTION_LIMIT: 26.58 mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); 26.59 break; 26.60 26.61 - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: 26.62 + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: 26.63 mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); 26.64 break; 26.65 26.66 - case HttpRequest::CP_ENABLE_PIPELINING: 26.67 + case HttpRequest::PO_ENABLE_PIPELINING: 26.68 mPipelining = llclamp(value, 0L, 1L); 26.69 break; 26.70 26.71 + case HttpRequest::PO_THROTTLE_RATE: 26.72 + mThrottleRate = llclamp(value, 0L, 1000000L); 26.73 + break; 26.74 + 26.75 default: 26.76 return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); 26.77 } 26.78 26.79 - mSetMask |= 1UL << int(opt); 26.80 return HttpStatus(); 26.81 } 26.82 26.83 26.84 -HttpStatus HttpPolicyClass::get(HttpRequest::EClassPolicy opt, long * value) 26.85 +HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) const 26.86 { 26.87 - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); 26.88 - long * src(NULL); 26.89 - 26.90 switch (opt) 26.91 { 26.92 - case HttpRequest::CP_CONNECTION_LIMIT: 26.93 - src = &mConnectionLimit; 26.94 + case HttpRequest::PO_CONNECTION_LIMIT: 26.95 + *value = mConnectionLimit; 26.96 break; 26.97 26.98 - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: 26.99 - src = &mPerHostConnectionLimit; 26.100 + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: 26.101 + *value = mPerHostConnectionLimit; 26.102 break; 26.103 26.104 - case HttpRequest::CP_ENABLE_PIPELINING: 26.105 - src = &mPipelining; 26.106 + case HttpRequest::PO_ENABLE_PIPELINING: 26.107 + *value = mPipelining; 26.108 + break; 26.109 + 26.110 + case HttpRequest::PO_THROTTLE_RATE: 26.111 + *value = mThrottleRate; 26.112 break; 26.113 26.114 default: 26.115 return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); 26.116 } 26.117 26.118 - if (! (mSetMask & (1UL << int(opt)))) 26.119 - return not_set; 26.120 - 26.121 - *value = *src; 26.122 return HttpStatus(); 26.123 } 26.124
27.1 --- a/indra/llcorehttp/_httppolicyclass.h Mon Feb 24 11:33:41 2014 -0500 27.2 +++ b/indra/llcorehttp/_httppolicyclass.h Tue Feb 25 13:25:40 2014 -0500 27.3 @@ -4,7 +4,7 @@ 27.4 * 27.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 27.6 * Second Life Viewer Source Code 27.7 - * Copyright (C) 2012, Linden Research, Inc. 27.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 27.9 * 27.10 * This library is free software; you can redistribute it and/or 27.11 * modify it under the terms of the GNU Lesser General Public 27.12 @@ -34,6 +34,18 @@ 27.13 namespace LLCore 27.14 { 27.15 27.16 +/// Options struct for per-class policy options. 27.17 +/// 27.18 +/// Combines both raw blob data access with semantics-enforcing 27.19 +/// set/get interfaces. For internal operations by the worker 27.20 +/// thread, just grab the setting directly from instance and test/use 27.21 +/// as needed. When attached to external APIs (the public API 27.22 +/// options interfaces) the set/get methods are available to 27.23 +/// enforce correct ranges, data types, contexts, etc. and suitable 27.24 +/// status values are returned. 27.25 +/// 27.26 +/// Threading: Single-threaded. In practice, init thread before 27.27 +/// worker starts, worker thread after. 27.28 class HttpPolicyClass 27.29 { 27.30 public: 27.31 @@ -44,14 +56,14 @@ 27.32 HttpPolicyClass(const HttpPolicyClass &); // Not defined 27.33 27.34 public: 27.35 - HttpStatus set(HttpRequest::EClassPolicy opt, long value); 27.36 - HttpStatus get(HttpRequest::EClassPolicy opt, long * value); 27.37 + HttpStatus set(HttpRequest::EPolicyOption opt, long value); 27.38 + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; 27.39 27.40 public: 27.41 - unsigned long mSetMask; 27.42 long mConnectionLimit; 27.43 long mPerHostConnectionLimit; 27.44 long mPipelining; 27.45 + long mThrottleRate; 27.46 }; // end class HttpPolicyClass 27.47 27.48 } // end namespace LLCore
28.1 --- a/indra/llcorehttp/_httppolicyglobal.cpp Mon Feb 24 11:33:41 2014 -0500 28.2 +++ b/indra/llcorehttp/_httppolicyglobal.cpp Tue Feb 25 13:25:40 2014 -0500 28.3 @@ -4,7 +4,7 @@ 28.4 * 28.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 28.6 * Second Life Viewer Source Code 28.7 - * Copyright (C) 2012, Linden Research, Inc. 28.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 28.9 * 28.10 * This library is free software; you can redistribute it and/or 28.11 * modify it under the terms of the GNU Lesser General Public 28.12 @@ -34,8 +34,7 @@ 28.13 28.14 28.15 HttpPolicyGlobal::HttpPolicyGlobal() 28.16 - : mSetMask(0UL), 28.17 - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), 28.18 + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), 28.19 mTrace(HTTP_TRACE_OFF), 28.20 mUseLLProxy(0) 28.21 {} 28.22 @@ -49,7 +48,6 @@ 28.23 { 28.24 if (this != &other) 28.25 { 28.26 - mSetMask = other.mSetMask; 28.27 mConnectionLimit = other.mConnectionLimit; 28.28 mCAPath = other.mCAPath; 28.29 mCAFile = other.mCAFile; 28.30 @@ -61,19 +59,19 @@ 28.31 } 28.32 28.33 28.34 -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) 28.35 +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, long value) 28.36 { 28.37 switch (opt) 28.38 { 28.39 - case HttpRequest::GP_CONNECTION_LIMIT: 28.40 + case HttpRequest::PO_CONNECTION_LIMIT: 28.41 mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); 28.42 break; 28.43 28.44 - case HttpRequest::GP_TRACE: 28.45 + case HttpRequest::PO_TRACE: 28.46 mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX)); 28.47 break; 28.48 28.49 - case HttpRequest::GP_LLPROXY: 28.50 + case HttpRequest::PO_LLPROXY: 28.51 mUseLLProxy = llclamp(value, 0L, 1L); 28.52 break; 28.53 28.54 @@ -81,24 +79,23 @@ 28.55 return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); 28.56 } 28.57 28.58 - mSetMask |= 1UL << int(opt); 28.59 return HttpStatus(); 28.60 } 28.61 28.62 28.63 -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::string & value) 28.64 +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, const std::string & value) 28.65 { 28.66 switch (opt) 28.67 { 28.68 - case HttpRequest::GP_CA_PATH: 28.69 + case HttpRequest::PO_CA_PATH: 28.70 mCAPath = value; 28.71 break; 28.72 28.73 - case HttpRequest::GP_CA_FILE: 28.74 + case HttpRequest::PO_CA_FILE: 28.75 mCAFile = value; 28.76 break; 28.77 28.78 - case HttpRequest::GP_HTTP_PROXY: 28.79 + case HttpRequest::PO_HTTP_PROXY: 28.80 mCAFile = value; 28.81 break; 28.82 28.83 @@ -106,69 +103,54 @@ 28.84 return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); 28.85 } 28.86 28.87 - mSetMask |= 1UL << int(opt); 28.88 return HttpStatus(); 28.89 } 28.90 28.91 28.92 -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) 28.93 +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, long * value) const 28.94 { 28.95 - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); 28.96 - long * src(NULL); 28.97 - 28.98 switch (opt) 28.99 { 28.100 - case HttpRequest::GP_CONNECTION_LIMIT: 28.101 - src = &mConnectionLimit; 28.102 + case HttpRequest::PO_CONNECTION_LIMIT: 28.103 + *value = mConnectionLimit; 28.104 break; 28.105 28.106 - case HttpRequest::GP_TRACE: 28.107 - src = &mTrace; 28.108 + case HttpRequest::PO_TRACE: 28.109 + *value = mTrace; 28.110 break; 28.111 28.112 - case HttpRequest::GP_LLPROXY: 28.113 - src = &mUseLLProxy; 28.114 + case HttpRequest::PO_LLPROXY: 28.115 + *value = mUseLLProxy; 28.116 break; 28.117 28.118 default: 28.119 return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); 28.120 } 28.121 28.122 - if (! (mSetMask & (1UL << int(opt)))) 28.123 - return not_set; 28.124 - 28.125 - *value = *src; 28.126 return HttpStatus(); 28.127 } 28.128 28.129 28.130 -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) 28.131 +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, std::string * value) const 28.132 { 28.133 - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); 28.134 - const std::string * src(NULL); 28.135 - 28.136 switch (opt) 28.137 { 28.138 - case HttpRequest::GP_CA_PATH: 28.139 - src = &mCAPath; 28.140 + case HttpRequest::PO_CA_PATH: 28.141 + *value = mCAPath; 28.142 break; 28.143 28.144 - case HttpRequest::GP_CA_FILE: 28.145 - src = &mCAFile; 28.146 + case HttpRequest::PO_CA_FILE: 28.147 + *value = mCAFile; 28.148 break; 28.149 28.150 - case HttpRequest::GP_HTTP_PROXY: 28.151 - src = &mHttpProxy; 28.152 + case HttpRequest::PO_HTTP_PROXY: 28.153 + *value = mHttpProxy; 28.154 break; 28.155 28.156 default: 28.157 return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); 28.158 } 28.159 28.160 - if (! (mSetMask & (1UL << int(opt)))) 28.161 - return not_set; 28.162 - 28.163 - *value = src; 28.164 return HttpStatus(); 28.165 } 28.166
29.1 --- a/indra/llcorehttp/_httppolicyglobal.h Mon Feb 24 11:33:41 2014 -0500 29.2 +++ b/indra/llcorehttp/_httppolicyglobal.h Tue Feb 25 13:25:40 2014 -0500 29.3 @@ -4,7 +4,7 @@ 29.4 * 29.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 29.6 * Second Life Viewer Source Code 29.7 - * Copyright (C) 2012, Linden Research, Inc. 29.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 29.9 * 29.10 * This library is free software; you can redistribute it and/or 29.11 * modify it under the terms of the GNU Lesser General Public 29.12 @@ -34,6 +34,18 @@ 29.13 namespace LLCore 29.14 { 29.15 29.16 +/// Options struct for global policy options. 29.17 +/// 29.18 +/// Combines both raw blob data access with semantics-enforcing 29.19 +/// set/get interfaces. For internal operations by the worker 29.20 +/// thread, just grab the setting directly from instance and test/use 29.21 +/// as needed. When attached to external APIs (the public API 29.22 +/// options interfaces) the set/get methods are available to 29.23 +/// enforce correct ranges, data types, contexts, etc. and suitable 29.24 +/// status values are returned. 29.25 +/// 29.26 +/// Threading: Single-threaded. In practice, init thread before 29.27 +/// worker starts, worker thread after. 29.28 class HttpPolicyGlobal 29.29 { 29.30 public: 29.31 @@ -46,13 +58,12 @@ 29.32 HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined 29.33 29.34 public: 29.35 - HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); 29.36 - HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); 29.37 - HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); 29.38 - HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); 29.39 + HttpStatus set(HttpRequest::EPolicyOption opt, long value); 29.40 + HttpStatus set(HttpRequest::EPolicyOption opt, const std::string & value); 29.41 + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; 29.42 + HttpStatus get(HttpRequest::EPolicyOption opt, std::string * value) const; 29.43 29.44 public: 29.45 - unsigned long mSetMask; 29.46 long mConnectionLimit; 29.47 std::string mCAPath; 29.48 std::string mCAFile;
30.1 --- a/indra/llcorehttp/_httpservice.cpp Mon Feb 24 11:33:41 2014 -0500 30.2 +++ b/indra/llcorehttp/_httpservice.cpp Tue Feb 25 13:25:40 2014 -0500 30.3 @@ -4,7 +4,7 @@ 30.4 * 30.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 30.6 * Second Life Viewer Source Code 30.7 - * Copyright (C) 2012, Linden Research, Inc. 30.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 30.9 * 30.10 * This library is free software; you can redistribute it and/or 30.11 * modify it under the terms of the GNU Lesser General Public 30.12 @@ -43,6 +43,18 @@ 30.13 namespace LLCore 30.14 { 30.15 30.16 +const HttpService::OptionDescriptor HttpService::sOptionDesc[] = 30.17 +{ // isLong isDynamic isGlobal isClass 30.18 + { true, true, true, true }, // PO_CONNECTION_LIMIT 30.19 + { true, true, false, true }, // PO_PER_HOST_CONNECTION_LIMIT 30.20 + { false, false, true, false }, // PO_CA_PATH 30.21 + { false, false, true, false }, // PO_CA_FILE 30.22 + { false, true, true, false }, // PO_HTTP_PROXY 30.23 + { true, true, true, false }, // PO_LLPROXY 30.24 + { true, true, true, false }, // PO_TRACE 30.25 + { true, true, false, true }, // PO_ENABLE_PIPELINING 30.26 + { true, true, false, true } // PO_THROTTLE_RATE 30.27 +}; 30.28 HttpService * HttpService::sInstance(NULL); 30.29 volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); 30.30 30.31 @@ -51,15 +63,9 @@ 30.32 mExitRequested(0U), 30.33 mThread(NULL), 30.34 mPolicy(NULL), 30.35 - mTransport(NULL) 30.36 -{ 30.37 - // Create the default policy class 30.38 - HttpPolicyClass pol_class; 30.39 - pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); 30.40 - pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); 30.41 - pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); 30.42 - mPolicyClasses.push_back(pol_class); 30.43 -} 30.44 + mTransport(NULL), 30.45 + mLastPolicy(0) 30.46 +{} 30.47 30.48 30.49 HttpService::~HttpService() 30.50 @@ -149,13 +155,8 @@ 30.51 30.52 HttpRequest::policy_t HttpService::createPolicyClass() 30.53 { 30.54 - const HttpRequest::policy_t policy_class(mPolicyClasses.size()); 30.55 - if (policy_class >= HTTP_POLICY_CLASS_LIMIT) 30.56 - { 30.57 - return 0; 30.58 - } 30.59 - mPolicyClasses.push_back(HttpPolicyClass()); 30.60 - return policy_class; 30.61 + mLastPolicy = mPolicy->createPolicyClass(); 30.62 + return mLastPolicy; 30.63 } 30.64 30.65 30.66 @@ -188,8 +189,8 @@ 30.67 } 30.68 30.69 // Push current policy definitions, enable policy & transport components 30.70 - mPolicy->start(mPolicyGlobal, mPolicyClasses); 30.71 - mTransport->start(mPolicyClasses.size()); 30.72 + mPolicy->start(); 30.73 + mTransport->start(mLastPolicy + 1); 30.74 30.75 mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1)); 30.76 sState = RUNNING; 30.77 @@ -322,7 +323,7 @@ 30.78 { 30.79 // Setup for subsequent tracing 30.80 long tracing(HTTP_TRACE_OFF); 30.81 - mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); 30.82 + mPolicy->getGlobalOptions().get(HttpRequest::PO_TRACE, &tracing); 30.83 op->mTracing = (std::max)(op->mTracing, int(tracing)); 30.84 30.85 if (op->mTracing > HTTP_TRACE_OFF) 30.86 @@ -345,4 +346,137 @@ 30.87 } 30.88 30.89 30.90 +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, 30.91 + long * ret_value) 30.92 +{ 30.93 + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range 30.94 + || opt >= HttpRequest::PO_LAST // ditto 30.95 + || (! sOptionDesc[opt].mIsLong) // datatype is long 30.96 + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range 30.97 + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted 30.98 + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted 30.99 + // can always get, no dynamic check 30.100 + { 30.101 + return HttpStatus(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); 30.102 + } 30.103 + 30.104 + HttpStatus status; 30.105 + if (pclass == HttpRequest::GLOBAL_POLICY_ID) 30.106 + { 30.107 + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); 30.108 + 30.109 + status = opts.get(opt, ret_value); 30.110 + } 30.111 + else 30.112 + { 30.113 + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); 30.114 + 30.115 + status = opts.get(opt, ret_value); 30.116 + } 30.117 + 30.118 + return status; 30.119 +} 30.120 + 30.121 + 30.122 +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, 30.123 + std::string * ret_value) 30.124 +{ 30.125 + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); 30.126 + 30.127 + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range 30.128 + || opt >= HttpRequest::PO_LAST // ditto 30.129 + || (sOptionDesc[opt].mIsLong) // datatype is string 30.130 + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range 30.131 + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted 30.132 + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted 30.133 + // can always get, no dynamic check 30.134 + { 30.135 + return status; 30.136 + } 30.137 + 30.138 + // Only global has string values 30.139 + if (pclass == HttpRequest::GLOBAL_POLICY_ID) 30.140 + { 30.141 + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); 30.142 + 30.143 + status = opts.get(opt, ret_value); 30.144 + } 30.145 + 30.146 + return status; 30.147 +} 30.148 + 30.149 + 30.150 +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, 30.151 + long value, long * ret_value) 30.152 +{ 30.153 + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); 30.154 + 30.155 + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range 30.156 + || opt >= HttpRequest::PO_LAST // ditto 30.157 + || (! sOptionDesc[opt].mIsLong) // datatype is long 30.158 + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range 30.159 + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted 30.160 + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted 30.161 + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted 30.162 + { 30.163 + return status; 30.164 + } 30.165 + 30.166 + if (pclass == HttpRequest::GLOBAL_POLICY_ID) 30.167 + { 30.168 + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); 30.169 + 30.170 + status = opts.set(opt, value); 30.171 + if (status && ret_value) 30.172 + { 30.173 + status = opts.get(opt, ret_value); 30.174 + } 30.175 + } 30.176 + else 30.177 + { 30.178 + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); 30.179 + 30.180 + status = opts.set(opt, value); 30.181 + if (status && ret_value) 30.182 + { 30.183 + status = opts.get(opt, ret_value); 30.184 + } 30.185 + } 30.186 + 30.187 + return status; 30.188 +} 30.189 + 30.190 + 30.191 +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, 30.192 + const std::string & value, std::string * ret_value) 30.193 +{ 30.194 + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); 30.195 + 30.196 + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range 30.197 + || opt >= HttpRequest::PO_LAST // ditto 30.198 + || (sOptionDesc[opt].mIsLong) // datatype is string 30.199 + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range 30.200 + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted 30.201 + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted 30.202 + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted 30.203 + { 30.204 + return status; 30.205 + } 30.206 + 30.207 + // Only string values are global at this time 30.208 + if (pclass == HttpRequest::GLOBAL_POLICY_ID) 30.209 + { 30.210 + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); 30.211 + 30.212 + status = opts.set(opt, value); 30.213 + if (status && ret_value) 30.214 + { 30.215 + status = opts.get(opt, ret_value); 30.216 + } 30.217 + } 30.218 + 30.219 + return status; 30.220 +} 30.221 + 30.222 + 30.223 } // end namespace LLCore
31.1 --- a/indra/llcorehttp/_httpservice.h Mon Feb 24 11:33:41 2014 -0500 31.2 +++ b/indra/llcorehttp/_httpservice.h Tue Feb 25 13:25:40 2014 -0500 31.3 @@ -4,7 +4,7 @@ 31.4 * 31.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 31.6 * Second Life Viewer Source Code 31.7 - * Copyright (C) 2012, Linden Research, Inc. 31.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 31.9 * 31.10 * This library is free software; you can redistribute it and/or 31.11 * modify it under the terms of the GNU Lesser General Public 31.12 @@ -53,6 +53,7 @@ 31.13 class HttpRequestQueue; 31.14 class HttpPolicy; 31.15 class HttpLibcurl; 31.16 +class HttpOpSetGet; 31.17 31.18 31.19 /// The HttpService class does the work behind the request queue. It 31.20 @@ -106,7 +107,7 @@ 31.21 NORMAL, ///< continuous polling of request, ready, active queues 31.22 REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write 31.23 }; 31.24 - 31.25 + 31.26 static void init(HttpRequestQueue *); 31.27 static void term(); 31.28 31.29 @@ -136,7 +137,7 @@ 31.30 /// acquires its weaknesses. 31.31 static bool isStopped(); 31.32 31.33 - /// Threading: callable by consumer thread *once*. 31.34 + /// Threading: callable by init thread *once*. 31.35 void startThread(); 31.36 31.37 /// Threading: callable by worker thread. 31.38 @@ -181,27 +182,38 @@ 31.39 } 31.40 31.41 /// Threading: callable by consumer thread. 31.42 - HttpPolicyGlobal & getGlobalOptions() 31.43 - { 31.44 - return mPolicyGlobal; 31.45 - } 31.46 - 31.47 - /// Threading: callable by consumer thread. 31.48 HttpRequest::policy_t createPolicyClass(); 31.49 31.50 - /// Threading: callable by consumer thread. 31.51 - HttpPolicyClass & getClassOptions(HttpRequest::policy_t policy_class) 31.52 - { 31.53 - llassert(policy_class >= 0 && policy_class < mPolicyClasses.size()); 31.54 - return mPolicyClasses[policy_class]; 31.55 - } 31.56 - 31.57 protected: 31.58 void threadRun(LLCoreInt::HttpThread * thread); 31.59 31.60 ELoopSpeed processRequestQueue(ELoopSpeed loop); 31.61 + 31.62 +protected: 31.63 + friend class HttpOpSetGet; 31.64 + friend class HttpRequest; 31.65 + 31.66 + // Used internally to describe what operations are allowed 31.67 + // on each policy option. 31.68 + struct OptionDescriptor 31.69 + { 31.70 + bool mIsLong; 31.71 + bool mIsDynamic; 31.72 + bool mIsGlobal; 31.73 + bool mIsClass; 31.74 + }; 31.75 + 31.76 + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, 31.77 + long * ret_value); 31.78 + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, 31.79 + std::string * ret_value); 31.80 + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, 31.81 + long value, long * ret_value); 31.82 + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, 31.83 + const std::string & value, std::string * ret_value); 31.84 31.85 protected: 31.86 + static const OptionDescriptor sOptionDesc[HttpRequest::PO_LAST]; 31.87 static HttpService * sInstance; 31.88 31.89 // === shared data === 31.90 @@ -210,13 +222,13 @@ 31.91 LLAtomicU32 mExitRequested; 31.92 LLCoreInt::HttpThread * mThread; 31.93 31.94 - // === consumer-thread-only data === 31.95 - HttpPolicyGlobal mPolicyGlobal; 31.96 - std::vector<HttpPolicyClass> mPolicyClasses; 31.97 - 31.98 // === working-thread-only data === 31.99 HttpPolicy * mPolicy; // Simple pointer, has ownership 31.100 HttpLibcurl * mTransport; // Simple pointer, has ownership 31.101 + 31.102 + // === main-thread-only data === 31.103 + HttpRequest::policy_t mLastPolicy; 31.104 + 31.105 }; // end class HttpService 31.106 31.107 } // end namespace LLCore
32.1 --- a/indra/llcorehttp/examples/http_texture_load.cpp Mon Feb 24 11:33:41 2014 -0500 32.2 +++ b/indra/llcorehttp/examples/http_texture_load.cpp Tue Feb 25 13:25:40 2014 -0500 32.3 @@ -39,6 +39,7 @@ 32.4 #include "httprequest.h" 32.5 #include "httphandler.h" 32.6 #include "httpresponse.h" 32.7 +#include "httpoptions.h" 32.8 #include "httpheaders.h" 32.9 #include "bufferarray.h" 32.10 #include "_mutex.h" 32.11 @@ -57,6 +58,7 @@ 32.12 32.13 // Default command line settings 32.14 static int concurrency_limit(40); 32.15 +static int highwater(100); 32.16 static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; 32.17 32.18 #if defined(WIN32) 32.19 @@ -79,11 +81,11 @@ 32.20 WorkingSet(); 32.21 ~WorkingSet(); 32.22 32.23 - bool reload(LLCore::HttpRequest *); 32.24 + bool reload(LLCore::HttpRequest *, LLCore::HttpOptions *); 32.25 32.26 virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); 32.27 32.28 - void loadTextureUuids(FILE * in); 32.29 + void loadAssetUuids(FILE * in); 32.30 32.31 public: 32.32 struct Spec 32.33 @@ -93,24 +95,27 @@ 32.34 int mLength; 32.35 }; 32.36 typedef std::set<LLCore::HttpHandle> handle_set_t; 32.37 - typedef std::vector<Spec> texture_list_t; 32.38 + typedef std::vector<Spec> asset_list_t; 32.39 32.40 public: 32.41 bool mVerbose; 32.42 bool mRandomRange; 32.43 - int mMaxConcurrency; 32.44 + int mRequestLowWater; 32.45 + int mRequestHighWater; 32.46 handle_set_t mHandles; 32.47 int mRemaining; 32.48 int mLimit; 32.49 int mAt; 32.50 std::string mUrl; 32.51 - texture_list_t mTextures; 32.52 + asset_list_t mAssets; 32.53 int mErrorsApi; 32.54 int mErrorsHttp; 32.55 int mErrorsHttp404; 32.56 int mErrorsHttp416; 32.57 int mErrorsHttp500; 32.58 int mErrorsHttp503; 32.59 + int mRetries; 32.60 + int mRetriesHttp503; 32.61 int mSuccesses; 32.62 long mByteCount; 32.63 LLCore::HttpHeaders * mHeaders; 32.64 @@ -158,7 +163,7 @@ 32.65 bool do_verbose(false); 32.66 32.67 int option(-1); 32.68 - while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) 32.69 + while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:"))) 32.70 { 32.71 switch (option) 32.72 { 32.73 @@ -182,6 +187,21 @@ 32.74 } 32.75 break; 32.76 32.77 + case 'H': 32.78 + { 32.79 + unsigned long value; 32.80 + char * end; 32.81 + 32.82 + value = strtoul(optarg, &end, 10); 32.83 + if (value < 1 || value > 100 || *end != '\0') 32.84 + { 32.85 + usage(std::cerr); 32.86 + return 1; 32.87 + } 32.88 + highwater = value; 32.89 + } 32.90 + break; 32.91 + 32.92 case 'R': 32.93 do_random = true; 32.94 break; 32.95 @@ -216,25 +236,32 @@ 32.96 // Initialization 32.97 init_curl(); 32.98 LLCore::HttpRequest::createService(); 32.99 - LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID, 32.100 - LLCore::HttpRequest::CP_CONNECTION_LIMIT, 32.101 - concurrency_limit); 32.102 + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, 32.103 + LLCore::HttpRequest::DEFAULT_POLICY_ID, 32.104 + concurrency_limit, 32.105 + NULL); 32.106 LLCore::HttpRequest::startThread(); 32.107 32.108 // Get service point 32.109 LLCore::HttpRequest * hr = new LLCore::HttpRequest(); 32.110 32.111 + // Get request options 32.112 + LLCore::HttpOptions * opt = new LLCore::HttpOptions(); 32.113 + opt->setRetries(12); 32.114 + opt->setUseRetryAfter(true); 32.115 + 32.116 // Get a handler/working set 32.117 WorkingSet ws; 32.118 32.119 // Fill the working set with work 32.120 ws.mUrl = url_format; 32.121 - ws.loadTextureUuids(uuids); 32.122 + ws.loadAssetUuids(uuids); 32.123 ws.mRandomRange = do_random; 32.124 ws.mVerbose = do_verbose; 32.125 - ws.mMaxConcurrency = 100; 32.126 + ws.mRequestHighWater = highwater; 32.127 + ws.mRequestLowWater = ws.mRequestHighWater / 2; 32.128 32.129 - if (! ws.mTextures.size()) 32.130 + if (! ws.mAssets.size()) 32.131 { 32.132 std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; 32.133 return 1; 32.134 @@ -246,9 +273,9 @@ 32.135 32.136 // Run it 32.137 int passes(0); 32.138 - while (! ws.reload(hr)) 32.139 + while (! ws.reload(hr, opt)) 32.140 { 32.141 - hr->update(5000000); 32.142 + hr->update(0); 32.143 ms_sleep(2); 32.144 if (0 == (++passes % 200)) 32.145 { 32.146 @@ -265,6 +292,8 @@ 32.147 std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << " HTTP 416 errors: " << ws.mErrorsHttp416 32.148 << " HTTP 500 errors: " << ws.mErrorsHttp500 << " HTTP 503 errors: " << ws.mErrorsHttp503 32.149 << std::endl; 32.150 + std::cout << "Retries: " << ws.mRetries << " Retries on 503: " << ws.mRetriesHttp503 32.151 + << std::endl; 32.152 std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime) 32.153 << " uS System CPU: " << (metrics.mEndSTime - metrics.mStartSTime) 32.154 << " uS Wall Time: " << (metrics.mEndWallTime - metrics.mStartWallTime) 32.155 @@ -275,6 +304,8 @@ 32.156 // Clean up 32.157 hr->requestStopThread(NULL); 32.158 ms_sleep(1000); 32.159 + opt->release(); 32.160 + opt = NULL; 32.161 delete hr; 32.162 LLCore::HttpRequest::destroyService(); 32.163 term_curl(); 32.164 @@ -300,8 +331,10 @@ 32.165 " -u <url_format> printf-style format string for URL generation\n" 32.166 " Default: " << url_format << "\n" 32.167 " -R Issue GETs with random Range: headers\n" 32.168 - " -c <limit> Maximum request concurrency. Range: [1..100]\n" 32.169 + " -c <limit> Maximum connection concurrency. Range: [1..100]\n" 32.170 " Default: " << concurrency_limit << "\n" 32.171 + " -H <limit> HTTP request highwater (requests fed to llcorehttp).\n" 32.172 + " Range: [1..100] Default: " << highwater << "\n" 32.173 " -v Verbose mode. Issue some chatter while running\n" 32.174 " -h print this help\n" 32.175 "\n" 32.176 @@ -322,10 +355,12 @@ 32.177 mErrorsHttp416(0), 32.178 mErrorsHttp500(0), 32.179 mErrorsHttp503(0), 32.180 + mRetries(0), 32.181 + mRetriesHttp503(0), 32.182 mSuccesses(0), 32.183 mByteCount(0L) 32.184 { 32.185 - mTextures.reserve(30000); 32.186 + mAssets.reserve(30000); 32.187 32.188 mHeaders = new LLCore::HttpHeaders; 32.189 mHeaders->append("Accept", "image/x-j2c"); 32.190 @@ -342,29 +377,35 @@ 32.191 } 32.192 32.193 32.194 -bool WorkingSet::reload(LLCore::HttpRequest * hr) 32.195 +bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) 32.196 { 32.197 - int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); 32.198 + if (mRequestLowWater <= mHandles.size()) 32.199 + { 32.200 + // Haven't fallen below low-water level yet. 32.201 + return false; 32.202 + } 32.203 + 32.204 + int to_do((std::min)(mRemaining, mRequestHighWater - int(mHandles.size()))); 32.205 32.206 for (int i(0); i < to_do; ++i) 32.207 { 32.208 char buffer[1024]; 32.209 #if defined(WIN32) 32.210 - _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); 32.211 + _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mAssets[mAt].mUuid.c_str()); 32.212 #else 32.213 - snprintf(buffer, sizeof(buffer), mUrl.c_str(), mTextures[mAt].mUuid.c_str()); 32.214 + snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); 32.215 #endif 32.216 - int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); 32.217 - int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); 32.218 + int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); 32.219 + int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength); 32.220 32.221 LLCore::HttpHandle handle; 32.222 if (offset || length) 32.223 { 32.224 - handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, mHeaders, this); 32.225 + handle = hr->requestGetByteRange(0, 0, buffer, offset, length, opt, mHeaders, this); 32.226 } 32.227 else 32.228 { 32.229 - handle = hr->requestGet(0, 0, buffer, NULL, mHeaders, this); 32.230 + handle = hr->requestGet(0, 0, buffer, opt, mHeaders, this); 32.231 } 32.232 if (! handle) 32.233 { 32.234 @@ -410,7 +451,7 @@ 32.235 { 32.236 // More success 32.237 LLCore::BufferArray * data(response->getBody()); 32.238 - mByteCount += data->size(); 32.239 + mByteCount += data ? data->size() : 0; 32.240 ++mSuccesses; 32.241 } 32.242 else 32.243 @@ -446,6 +487,10 @@ 32.244 ++mErrorsApi; 32.245 } 32.246 } 32.247 + unsigned int retry(0U), retry_503(0U); 32.248 + response->getRetries(&retry, &retry_503); 32.249 + mRetries += int(retry); 32.250 + mRetriesHttp503 += int(retry_503); 32.251 mHandles.erase(it); 32.252 } 32.253 32.254 @@ -459,21 +504,21 @@ 32.255 } 32.256 32.257 32.258 -void WorkingSet::loadTextureUuids(FILE * in) 32.259 +void WorkingSet::loadAssetUuids(FILE * in) 32.260 { 32.261 char buffer[1024]; 32.262 32.263 while (fgets(buffer, sizeof(buffer), in)) 32.264 { 32.265 - WorkingSet::Spec texture; 32.266 + WorkingSet::Spec asset; 32.267 char * state(NULL); 32.268 char * token = strtok_r(buffer, " \t\n,", &state); 32.269 if (token && 36 == strlen(token)) 32.270 { 32.271 // Close enough for this function 32.272 - texture.mUuid = token; 32.273 - texture.mOffset = 0; 32.274 - texture.mLength = 0; 32.275 + asset.mUuid = token; 32.276 + asset.mOffset = 0; 32.277 + asset.mLength = 0; 32.278 token = strtok_r(buffer, " \t\n,", &state); 32.279 if (token) 32.280 { 32.281 @@ -482,14 +527,14 @@ 32.282 if (token) 32.283 { 32.284 int length(atoi(token)); 32.285 - texture.mOffset = offset; 32.286 - texture.mLength = length; 32.287 + asset.mOffset = offset; 32.288 + asset.mLength = length; 32.289 } 32.290 } 32.291 - mTextures.push_back(texture); 32.292 + mAssets.push_back(asset); 32.293 } 32.294 } 32.295 - mRemaining = mLimit = mTextures.size(); 32.296 + mRemaining = mLimit = mAssets.size(); 32.297 } 32.298 32.299
33.1 --- a/indra/llcorehttp/httpcommon.cpp Mon Feb 24 11:33:41 2014 -0500 33.2 +++ b/indra/llcorehttp/httpcommon.cpp Tue Feb 25 13:25:40 2014 -0500 33.3 @@ -70,7 +70,8 @@ 33.4 "Invalid datatype for argument or option", 33.5 "Option has not been explicitly set", 33.6 "Option is not dynamic and must be set early", 33.7 - "Invalid HTTP status code received from server" 33.8 + "Invalid HTTP status code received from server", 33.9 + "Could not allocate required resource" 33.10 }; 33.11 static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); 33.12 33.13 @@ -177,6 +178,44 @@ 33.14 } 33.15 33.16 33.17 +std::string HttpStatus::toTerseString() const 33.18 +{ 33.19 + std::ostringstream result; 33.20 + 33.21 + unsigned int error_value((unsigned short) mStatus); 33.22 + 33.23 + switch (mType) 33.24 + { 33.25 + case EXT_CURL_EASY: 33.26 + result << "Easy_"; 33.27 + break; 33.28 + 33.29 + case EXT_CURL_MULTI: 33.30 + result << "Multi_"; 33.31 + break; 33.32 + 33.33 + case LLCORE: 33.34 + result << "Core_"; 33.35 + break; 33.36 + 33.37 + default: 33.38 + if (isHttpStatus()) 33.39 + { 33.40 + result << "Http_"; 33.41 + error_value = mType; 33.42 + } 33.43 + else 33.44 + { 33.45 + result << "Unknown_"; 33.46 + } 33.47 + break; 33.48 + } 33.49 + 33.50 + result << error_value; 33.51 + return result.str(); 33.52 +} 33.53 + 33.54 + 33.55 // Pass true on statuses that might actually be cleared by a 33.56 // retry. Library failures, calling problems, etc. aren't 33.57 // going to be fixed by squirting bits all over the Net. 33.58 @@ -206,6 +245,5 @@ 33.59 *this == inv_cont_range); // Short data read disagrees with content-range 33.60 } 33.61 33.62 - 33.63 } // end namespace LLCore 33.64
34.1 --- a/indra/llcorehttp/httpcommon.h Mon Feb 24 11:33:41 2014 -0500 34.2 +++ b/indra/llcorehttp/httpcommon.h Tue Feb 25 13:25:40 2014 -0500 34.3 @@ -29,9 +29,9 @@ 34.4 34.5 /// @package LLCore::HTTP 34.6 /// 34.7 -/// This library implements a high-level, Indra-code-free client interface to 34.8 -/// HTTP services based on actual patterns found in the viewer and simulator. 34.9 -/// Interfaces are similar to those supplied by the legacy classes 34.10 +/// This library implements a high-level, Indra-code-free (somewhat) client 34.11 +/// interface to HTTP services based on actual patterns found in the viewer 34.12 +/// and simulator. Interfaces are similar to those supplied by the legacy classes 34.13 /// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that 34.14 /// allows an application to specify connection behaviors: limits on 34.15 /// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. 34.16 @@ -52,7 +52,7 @@ 34.17 /// - "llcorehttp/httprequest.h" 34.18 /// - "llcorehttp/httpresponse.h" 34.19 /// 34.20 -/// The library is still under early development and particular users 34.21 +/// The library is still under development and particular users 34.22 /// may need access to internal implementation details that are found 34.23 /// in the _*.h header files. But this is a crutch to be avoided if at 34.24 /// all possible and probably indicates some interface work is neeeded. 34.25 @@ -66,6 +66,8 @@ 34.26 /// . CRYPTO_set_id_callback(...) 34.27 /// - HttpRequest::createService() called to instantiate singletons 34.28 /// and support objects. 34.29 +/// - HttpRequest::startThread() to kick off the worker thread and 34.30 +/// begin servicing requests. 34.31 /// 34.32 /// An HTTP consumer in an application, and an application may have many 34.33 /// consumers, does a few things: 34.34 @@ -91,10 +93,12 @@ 34.35 /// objects. 34.36 /// - Do completion processing in your onCompletion() method. 34.37 /// 34.38 -/// Code fragments: 34.39 -/// Rather than a poorly-maintained example in comments, look in the 34.40 -/// example subdirectory which is a minimal yet functional tool to do 34.41 -/// GET request performance testing. With four calls: 34.42 +/// Code fragments. 34.43 +/// 34.44 +/// Initialization. Rather than a poorly-maintained example in 34.45 +/// comments, look in the example subdirectory which is a minimal 34.46 +/// yet functional tool to do GET request performance testing. 34.47 +/// With four calls: 34.48 /// 34.49 /// init_curl(); 34.50 /// LLCore::HttpRequest::createService(); 34.51 @@ -103,7 +107,85 @@ 34.52 /// 34.53 /// the program is basically ready to issue requests. 34.54 /// 34.55 - 34.56 +/// HttpHandler. Having started life as a non-indra library, 34.57 +/// this code broke away from the classic Responder model and 34.58 +/// introduced a handler class to represent an interface for 34.59 +/// request responses. This is a non-reference-counted entity 34.60 +/// which can be used as a base class or a mixin. An instance 34.61 +/// of a handler can be used for each request or can be shared 34.62 +/// among any number of requests. Your choice but expect to 34.63 +/// code something like the following: 34.64 +/// 34.65 +/// class AppHandler : public LLCore::HttpHandler 34.66 +/// { 34.67 +/// public: 34.68 +/// virtual void onCompleted(HttpHandle handle, 34.69 +/// HttpResponse * response) 34.70 +/// { 34.71 +/// ... 34.72 +/// } 34.73 +/// ... 34.74 +/// }; 34.75 +/// ... 34.76 +/// handler = new handler(...); 34.77 +/// 34.78 +/// 34.79 +/// Issuing requests. Using 'hr' above, 34.80 +/// 34.81 +/// hr->requestGet(HttpRequest::DEFAULT_POLICY_ID, 34.82 +/// 0, // Priority, not used yet 34.83 +/// url, 34.84 +/// NULL, // options 34.85 +/// NULL, // additional headers 34.86 +/// handler); 34.87 +/// 34.88 +/// If that returns a value other than LLCORE_HTTP_HANDLE_INVALID, 34.89 +/// the request was successfully issued and there will eventally 34.90 +/// be a status delivered to the handler. If invalid is returnedd, 34.91 +/// the actual status can be retrieved by calling hr->getStatus(). 34.92 +/// 34.93 +/// Completing requests and delivering notifications. Operations 34.94 +/// are all performed by the worker thread and will be driven to 34.95 +/// completion regardless of caller actions. Notification of 34.96 +/// completion (success or failure) is done by calls to 34.97 +/// HttpRequest::update() which will invoke handlers for completed 34.98 +/// requests: 34.99 +/// 34.100 +/// hr->update(0); 34.101 +/// // Callbacks into handler->onCompleted() 34.102 +/// 34.103 +/// 34.104 +/// Threads. 34.105 +/// 34.106 +/// Threads are supported and used by this library. The various 34.107 +/// classes, methods and members are documented with thread 34.108 +/// constraints which programmers must follow and which are 34.109 +/// defined as follows: 34.110 +/// 34.111 +/// consumer Any thread that has instanced HttpRequest and is 34.112 +/// issuing requests. A particular instance can only 34.113 +/// be used by one consumer thread but a consumer may 34.114 +/// have many instances available to it. 34.115 +/// init Special consumer thread, usually the main thread, 34.116 +/// involved in setting up the library at startup. 34.117 +/// worker Thread used internally by the library to perform 34.118 +/// HTTP operations. Consumers will not have to deal 34.119 +/// with this thread directly but some APIs are reserved 34.120 +/// to it. 34.121 +/// any Consumer or worker thread. 34.122 +/// 34.123 +/// For the most part, API users will not have to do much in the 34.124 +/// way of ensuring thread safely. However, there is a tremendous 34.125 +/// amount of sharing between threads of read-only data. So when 34.126 +/// documentation declares that an option or header instance 34.127 +/// becomes shared between consumer and worker, the consumer must 34.128 +/// not modify the shared object. 34.129 +/// 34.130 +/// Internally, there is almost no thread synchronization. During 34.131 +/// normal operations (non-init, non-term), only the request queue 34.132 +/// and the multiple reply queues are shared between threads and 34.133 +/// only here are mutexes used. 34.134 +/// 34.135 34.136 #include "linden_common.h" // Modifies curl/curl.h interfaces 34.137 34.138 @@ -164,7 +246,10 @@ 34.139 HE_OPT_NOT_DYNAMIC = 8, 34.140 34.141 // Invalid HTTP status code returned by server 34.142 - HE_INVALID_HTTP_STATUS = 9 34.143 + HE_INVALID_HTTP_STATUS = 9, 34.144 + 34.145 + // Couldn't allocate resource, typically libcurl handle 34.146 + HE_BAD_ALLOC = 10 34.147 34.148 }; // end enum HttpError 34.149 34.150 @@ -239,9 +324,10 @@ 34.151 return *this; 34.152 } 34.153 34.154 - static const type_enum_t EXT_CURL_EASY = 0; 34.155 - static const type_enum_t EXT_CURL_MULTI = 1; 34.156 - static const type_enum_t LLCORE = 2; 34.157 + static const type_enum_t EXT_CURL_EASY = 0; ///< mStatus is an error from a curl_easy_*() call 34.158 + static const type_enum_t EXT_CURL_MULTI = 1; ///< mStatus is an error from a curl_multi_*() call 34.159 + static const type_enum_t LLCORE = 2; ///< mStatus is an HE_* error code 34.160 + ///< 100-999 directly represent HTTP status codes 34.161 34.162 type_enum_t mType; 34.163 short mStatus; 34.164 @@ -297,6 +383,14 @@ 34.165 /// LLCore itself). 34.166 std::string toString() const; 34.167 34.168 + /// Convert status to a compact string representation 34.169 + /// of the form: "<type>_<value>". The <type> will be 34.170 + /// one of: Core, Http, Easy, Multi, Unknown. And 34.171 + /// <value> will be an unsigned integer. More easily 34.172 + /// interpreted than the hex representation, it's still 34.173 + /// compact and easily searched. 34.174 + std::string toTerseString() const; 34.175 + 34.176 /// Returns true if the status value represents an 34.177 /// HTTP response status (100 - 999). 34.178 bool isHttpStatus() const
35.1 --- a/indra/llcorehttp/httpoptions.cpp Mon Feb 24 11:33:41 2014 -0500 35.2 +++ b/indra/llcorehttp/httpoptions.cpp Tue Feb 25 13:25:40 2014 -0500 35.3 @@ -4,7 +4,7 @@ 35.4 * 35.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 35.6 * Second Life Viewer Source Code 35.7 - * Copyright (C) 2012, Linden Research, Inc. 35.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 35.9 * 35.10 * This library is free software; you can redistribute it and/or 35.11 * modify it under the terms of the GNU Lesser General Public 35.12 @@ -38,7 +38,9 @@ 35.13 mWantHeaders(false), 35.14 mTracing(HTTP_TRACE_OFF), 35.15 mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), 35.16 - mRetries(HTTP_RETRY_COUNT_DEFAULT) 35.17 + mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), 35.18 + mRetries(HTTP_RETRY_COUNT_DEFAULT), 35.19 + mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT) 35.20 {} 35.21 35.22 35.23 @@ -64,10 +66,21 @@ 35.24 } 35.25 35.26 35.27 +void HttpOptions::setTransferTimeout(unsigned int timeout) 35.28 +{ 35.29 + mTransferTimeout = timeout; 35.30 +} 35.31 + 35.32 + 35.33 void HttpOptions::setRetries(unsigned int retries) 35.34 { 35.35 mRetries = retries; 35.36 } 35.37 35.38 +void HttpOptions::setUseRetryAfter(bool use_retry) 35.39 +{ 35.40 + mUseRetryAfter = use_retry; 35.41 +} 35.42 + 35.43 35.44 } // end namespace LLCore
36.1 --- a/indra/llcorehttp/httpoptions.h Mon Feb 24 11:33:41 2014 -0500 36.2 +++ b/indra/llcorehttp/httpoptions.h Tue Feb 25 13:25:40 2014 -0500 36.3 @@ -4,7 +4,7 @@ 36.4 * 36.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 36.6 * Second Life Viewer Source Code 36.7 - * Copyright (C) 2012, Linden Research, Inc. 36.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 36.9 * 36.10 * This library is free software; you can redistribute it and/or 36.11 * modify it under the terms of the GNU Lesser General Public 36.12 @@ -68,36 +68,55 @@ 36.13 void operator=(const HttpOptions &); // Not defined 36.14 36.15 public: 36.16 + // Default: false 36.17 void setWantHeaders(bool wanted); 36.18 bool getWantHeaders() const 36.19 { 36.20 return mWantHeaders; 36.21 } 36.22 - 36.23 + 36.24 + // Default: 0 36.25 void setTrace(int long); 36.26 int getTrace() const 36.27 { 36.28 return mTracing; 36.29 } 36.30 36.31 + // Default: 30 36.32 void setTimeout(unsigned int timeout); 36.33 unsigned int getTimeout() const 36.34 { 36.35 return mTimeout; 36.36 } 36.37 36.38 + // Default: 0 36.39 + void setTransferTimeout(unsigned int timeout); 36.40 + unsigned int getTransferTimeout() const 36.41 + { 36.42 + return mTransferTimeout; 36.43 + } 36.44 + 36.45 + // Default: 8 36.46 void setRetries(unsigned int retries); 36.47 unsigned int getRetries() const 36.48 { 36.49 return mRetries; 36.50 } 36.51 + 36.52 + // Default: true 36.53 + void setUseRetryAfter(bool use_retry); 36.54 + bool getUseRetryAfter() const 36.55 + { 36.56 + return mUseRetryAfter; 36.57 + } 36.58 36.59 protected: 36.60 bool mWantHeaders; 36.61 int mTracing; 36.62 unsigned int mTimeout; 36.63 + unsigned int mTransferTimeout; 36.64 unsigned int mRetries; 36.65 - 36.66 + bool mUseRetryAfter; 36.67 }; // end class HttpOptions 36.68 36.69
37.1 --- a/indra/llcorehttp/httprequest.cpp Mon Feb 24 11:33:41 2014 -0500 37.2 +++ b/indra/llcorehttp/httprequest.cpp Tue Feb 25 13:25:40 2014 -0500 37.3 @@ -4,7 +4,7 @@ 37.4 * 37.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 37.6 * Second Life Viewer Source Code 37.7 - * Copyright (C) 2012, Linden Research, Inc. 37.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 37.9 * 37.10 * This library is free software; you can redistribute it and/or 37.11 * modify it under the terms of the GNU Lesser General Public 37.12 @@ -54,12 +54,8 @@ 37.13 // ==================================== 37.14 37.15 37.16 -HttpRequest::policy_t HttpRequest::sNextPolicyID(1); 37.17 - 37.18 - 37.19 HttpRequest::HttpRequest() 37.20 - : //HttpHandler(), 37.21 - mReplyQueue(NULL), 37.22 + : mReplyQueue(NULL), 37.23 mRequestQueue(NULL) 37.24 { 37.25 mRequestQueue = HttpRequestQueue::instanceOf(); 37.26 @@ -90,26 +86,6 @@ 37.27 // ==================================== 37.28 37.29 37.30 -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) 37.31 -{ 37.32 - if (HttpService::RUNNING == HttpService::instanceOf()->getState()) 37.33 - { 37.34 - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); 37.35 - } 37.36 - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); 37.37 -} 37.38 - 37.39 - 37.40 -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) 37.41 -{ 37.42 - if (HttpService::RUNNING == HttpService::instanceOf()->getState()) 37.43 - { 37.44 - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); 37.45 - } 37.46 - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); 37.47 -} 37.48 - 37.49 - 37.50 HttpRequest::policy_t HttpRequest::createPolicyClass() 37.51 { 37.52 if (HttpService::RUNNING == HttpService::instanceOf()->getState()) 37.53 @@ -120,15 +96,81 @@ 37.54 } 37.55 37.56 37.57 -HttpStatus HttpRequest::setPolicyClassOption(policy_t policy_id, 37.58 - EClassPolicy opt, 37.59 - long value) 37.60 +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, 37.61 + long value, long * ret_value) 37.62 { 37.63 if (HttpService::RUNNING == HttpService::instanceOf()->getState()) 37.64 { 37.65 return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); 37.66 } 37.67 - return HttpService::instanceOf()->getClassOptions(policy_id).set(opt, value); 37.68 + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); 37.69 +} 37.70 + 37.71 + 37.72 +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, 37.73 + const std::string & value, std::string * ret_value) 37.74 +{ 37.75 + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) 37.76 + { 37.77 + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); 37.78 + } 37.79 + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); 37.80 +} 37.81 + 37.82 + 37.83 +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, 37.84 + long value, HttpHandler * handler) 37.85 +{ 37.86 + HttpStatus status; 37.87 + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); 37.88 + 37.89 + HttpOpSetGet * op = new HttpOpSetGet(); 37.90 + if (! (status = op->setupSet(opt, pclass, value))) 37.91 + { 37.92 + op->release(); 37.93 + mLastReqStatus = status; 37.94 + return handle; 37.95 + } 37.96 + op->setReplyPath(mReplyQueue, handler); 37.97 + if (! (status = mRequestQueue->addOp(op))) // transfers refcount 37.98 + { 37.99 + op->release(); 37.100 + mLastReqStatus = status; 37.101 + return handle; 37.102 + } 37.103 + 37.104 + mLastReqStatus = status; 37.105 + handle = static_cast<HttpHandle>(op); 37.106 + 37.107 + return handle; 37.108 +} 37.109 + 37.110 + 37.111 +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, 37.112 + const std::string & value, HttpHandler * handler) 37.113 +{ 37.114 + HttpStatus status; 37.115 + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); 37.116 + 37.117 + HttpOpSetGet * op = new HttpOpSetGet(); 37.118 + if (! (status = op->setupSet(opt, pclass, value))) 37.119 + { 37.120 + op->release(); 37.121 + mLastReqStatus = status; 37.122 + return handle; 37.123 + } 37.124 + op->setReplyPath(mReplyQueue, handler); 37.125 + if (! (status = mRequestQueue->addOp(op))) // transfers refcount 37.126 + { 37.127 + op->release(); 37.128 + mLastReqStatus = status; 37.129 + return handle; 37.130 + } 37.131 + 37.132 + mLastReqStatus = status; 37.133 + handle = static_cast<HttpHandle>(op); 37.134 + 37.135 + return handle; 37.136 } 37.137 37.138 37.139 @@ -474,31 +516,6 @@ 37.140 return handle; 37.141 } 37.142 37.143 -// ==================================== 37.144 -// Dynamic Policy Methods 37.145 -// ==================================== 37.146 - 37.147 -HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler) 37.148 -{ 37.149 - HttpStatus status; 37.150 - HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); 37.151 - 37.152 - HttpOpSetGet * op = new HttpOpSetGet(); 37.153 - op->setupSet(GP_HTTP_PROXY, proxy); 37.154 - op->setReplyPath(mReplyQueue, handler); 37.155 - if (! (status = mRequestQueue->addOp(op))) // transfers refcount 37.156 - { 37.157 - op->release(); 37.158 - mLastReqStatus = status; 37.159 - return handle; 37.160 - } 37.161 - 37.162 - mLastReqStatus = status; 37.163 - handle = static_cast<HttpHandle>(op); 37.164 - 37.165 - return handle; 37.166 -} 37.167 - 37.168 37.169 } // end namespace LLCore 37.170
38.1 --- a/indra/llcorehttp/httprequest.h Mon Feb 24 11:33:41 2014 -0500 38.2 +++ b/indra/llcorehttp/httprequest.h Tue Feb 25 13:25:40 2014 -0500 38.3 @@ -4,7 +4,7 @@ 38.4 * 38.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 38.6 * Second Life Viewer Source Code 38.7 - * Copyright (C) 2012, Linden Research, Inc. 38.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 38.9 * 38.10 * This library is free software; you can redistribute it and/or 38.11 * modify it under the terms of the GNU Lesser General Public 38.12 @@ -56,6 +56,9 @@ 38.13 /// The class supports the current HTTP request operations: 38.14 /// 38.15 /// - requestGetByteRange: GET with Range header for a single range of bytes 38.16 +/// - requestGet: 38.17 +/// - requestPost: 38.18 +/// - requestPut: 38.19 /// 38.20 /// Policy Classes 38.21 /// 38.22 @@ -100,9 +103,26 @@ 38.23 38.24 /// Represents a default, catch-all policy class that guarantees 38.25 /// eventual service for any HTTP request. 38.26 - static const int DEFAULT_POLICY_ID = 0; 38.27 + static const policy_t DEFAULT_POLICY_ID = 0; 38.28 + static const policy_t INVALID_POLICY_ID = 0xFFFFFFFFU; 38.29 + static const policy_t GLOBAL_POLICY_ID = 0xFFFFFFFEU; 38.30 38.31 - enum EGlobalPolicy 38.32 + /// Create a new policy class into which requests can be made. 38.33 + /// 38.34 + /// All class creation must occur before threads are started and 38.35 + /// transport begins. Policy classes are limited to a small value. 38.36 + /// Currently that limit is the default class + 1. 38.37 + /// 38.38 + /// @return If positive, the policy_id used to reference 38.39 + /// the class in other methods. If 0, requests 38.40 + /// for classes have exceeded internal limits 38.41 + /// or caller has tried to create a class after 38.42 + /// threads have been started. Caller must fallback 38.43 + /// and recover. 38.44 + /// 38.45 + static policy_t createPolicyClass(); 38.46 + 38.47 + enum EPolicyOption 38.48 { 38.49 /// Maximum number of connections the library will use to 38.50 /// perform operations. This is somewhat soft as the underlying 38.51 @@ -113,24 +133,40 @@ 38.52 /// a somewhat soft value. There may be an additional five 38.53 /// connections per policy class depending upon runtime 38.54 /// behavior. 38.55 - GP_CONNECTION_LIMIT, 38.56 + /// 38.57 + /// Both global and per-class 38.58 + PO_CONNECTION_LIMIT, 38.59 + 38.60 + /// Limits the number of connections used for a single 38.61 + /// literal address/port pair within the class. 38.62 + /// 38.63 + /// Per-class only 38.64 + PO_PER_HOST_CONNECTION_LIMIT, 38.65 38.66 /// String containing a system-appropriate directory name 38.67 /// where SSL certs are stored. 38.68 - GP_CA_PATH, 38.69 + /// 38.70 + /// Global only 38.71 + PO_CA_PATH, 38.72 38.73 /// String giving a full path to a file containing SSL certs. 38.74 - GP_CA_FILE, 38.75 + /// 38.76 + /// Global only 38.77 + PO_CA_FILE, 38.78 38.79 /// String of host/port to use as simple HTTP proxy. This is 38.80 /// going to change in the future into something more elaborate 38.81 /// that may support richer schemes. 38.82 - GP_HTTP_PROXY, 38.83 + /// 38.84 + /// Global only 38.85 + PO_HTTP_PROXY, 38.86 38.87 /// Long value that if non-zero enables the use of the 38.88 /// traditional LLProxy code for http/socks5 support. If 38.89 - /// enabled, has priority over GP_HTTP_PROXY. 38.90 - GP_LLPROXY, 38.91 + // enabled, has priority over GP_HTTP_PROXY. 38.92 + /// 38.93 + /// Global only 38.94 + PO_LLPROXY, 38.95 38.96 /// Long value setting the logging trace level for the 38.97 /// library. Possible values are: 38.98 @@ -143,50 +179,59 @@ 38.99 /// These values are also used in the trace modes for 38.100 /// individual requests in HttpOptions. Also be aware that 38.101 /// tracing tends to impact performance of the viewer. 38.102 - GP_TRACE 38.103 - }; 38.104 - 38.105 - /// Set a parameter on a global policy option. Calls 38.106 - /// made after the start of the servicing thread are 38.107 - /// not honored and return an error status. 38.108 - /// 38.109 - /// @param opt Enum of option to be set. 38.110 - /// @param value Desired value of option. 38.111 - /// @return Standard status code. 38.112 - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); 38.113 - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); 38.114 - 38.115 - /// Create a new policy class into which requests can be made. 38.116 - /// 38.117 - /// @return If positive, the policy_id used to reference 38.118 - /// the class in other methods. If 0, an error 38.119 - /// occurred and @see getStatus() may provide more 38.120 - /// detail on the reason. 38.121 - static policy_t createPolicyClass(); 38.122 - 38.123 - enum EClassPolicy 38.124 - { 38.125 - /// Limits the number of connections used for the class. 38.126 - CP_CONNECTION_LIMIT, 38.127 - 38.128 - /// Limits the number of connections used for a single 38.129 - /// literal address/port pair within the class. 38.130 - CP_PER_HOST_CONNECTION_LIMIT, 38.131 + /// 38.132 + /// Global only 38.133 + PO_TRACE, 38.134 38.135 /// Suitable requests are allowed to pipeline on their 38.136 /// connections when they ask for it. 38.137 - CP_ENABLE_PIPELINING 38.138 + /// 38.139 + /// Per-class only 38.140 + PO_ENABLE_PIPELINING, 38.141 + 38.142 + /// Controls whether client-side throttling should be 38.143 + /// performed on this policy class. Positive values 38.144 + /// enable throttling and specify the request rate 38.145 + /// (requests per second) that should be targetted. 38.146 + /// A value of zero, the default, specifies no throttling. 38.147 + /// 38.148 + /// Per-class only 38.149 + PO_THROTTLE_RATE, 38.150 + 38.151 + PO_LAST // Always at end 38.152 }; 38.153 - 38.154 + 38.155 + /// Set a policy option for a global or class parameter at 38.156 + /// startup time (prior to thread start). 38.157 + /// 38.158 + /// @param opt Enum of option to be set. 38.159 + /// @param pclass For class-based options, the policy class ID to 38.160 + /// be changed. For globals, specify GLOBAL_POLICY_ID. 38.161 + /// @param value Desired value of option. 38.162 + /// @param ret_value Pointer to receive effective set value 38.163 + /// if successful. May be NULL if effective 38.164 + /// value not wanted. 38.165 + /// @return Standard status code. 38.166 + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, 38.167 + long value, long * ret_value); 38.168 + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, 38.169 + const std::string & value, std::string * ret_value); 38.170 + 38.171 /// Set a parameter on a class-based policy option. Calls 38.172 /// made after the start of the servicing thread are 38.173 /// not honored and return an error status. 38.174 /// 38.175 - /// @param policy_id ID of class as returned by @see createPolicyClass(). 38.176 - /// @param opt Enum of option to be set. 38.177 - /// @param value Desired value of option. 38.178 - /// @return Standard status code. 38.179 - static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); 38.180 + /// @param opt Enum of option to be set. 38.181 + /// @param pclass For class-based options, the policy class ID to 38.182 + /// be changed. Ignored for globals but recommend 38.183 + /// using INVALID_POLICY_ID in this case. 38.184 + /// @param value Desired value of option. 38.185 + /// @return Handle of dynamic request. Use @see getStatus() if 38.186 + /// the returned handle is invalid. 38.187 + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, long value, 38.188 + HttpHandler * handler); 38.189 + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, const std::string & value, 38.190 + HttpHandler * handler); 38.191 38.192 /// @} 38.193 38.194 @@ -488,16 +533,6 @@ 38.195 38.196 /// @} 38.197 38.198 - /// @name DynamicPolicyMethods 38.199 - /// 38.200 - /// @{ 38.201 - 38.202 - /// Request that a running transport pick up a new proxy setting. 38.203 - /// An empty string will indicate no proxy is to be used. 38.204 - HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler); 38.205 - 38.206 - /// @} 38.207 - 38.208 protected: 38.209 void generateNotification(HttpOperation * op); 38.210 38.211 @@ -519,7 +554,6 @@ 38.212 /// Must be established before any threading is allowed to 38.213 /// start. 38.214 /// 38.215 - static policy_t sNextPolicyID; 38.216 38.217 /// @} 38.218 // End Global State
39.1 --- a/indra/llcorehttp/httpresponse.cpp Mon Feb 24 11:33:41 2014 -0500 39.2 +++ b/indra/llcorehttp/httpresponse.cpp Tue Feb 25 13:25:40 2014 -0500 39.3 @@ -4,7 +4,7 @@ 39.4 * 39.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 39.6 * Second Life Viewer Source Code 39.7 - * Copyright (C) 2012, Linden Research, Inc. 39.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 39.9 * 39.10 * This library is free software; you can redistribute it and/or 39.11 * modify it under the terms of the GNU Lesser General Public 39.12 @@ -39,7 +39,9 @@ 39.13 mReplyLength(0U), 39.14 mReplyFullLength(0U), 39.15 mBufferArray(NULL), 39.16 - mHeaders(NULL) 39.17 + mHeaders(NULL), 39.18 + mRetries(0U), 39.19 + m503Retries(0U) 39.20 {} 39.21 39.22
40.1 --- a/indra/llcorehttp/httpresponse.h Mon Feb 24 11:33:41 2014 -0500 40.2 +++ b/indra/llcorehttp/httpresponse.h Tue Feb 25 13:25:40 2014 -0500 40.3 @@ -4,7 +4,7 @@ 40.4 * 40.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 40.6 * Second Life Viewer Source Code 40.7 - * Copyright (C) 2012, Linden Research, Inc. 40.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 40.9 * 40.10 * This library is free software; you can redistribute it and/or 40.11 * modify it under the terms of the GNU Lesser General Public 40.12 @@ -149,6 +149,25 @@ 40.13 mContentType = con_type; 40.14 } 40.15 40.16 + /// Get and set retry attempt information on the request. 40.17 + void getRetries(unsigned int * retries, unsigned int * retries_503) const 40.18 + { 40.19 + if (retries) 40.20 + { 40.21 + *retries = mRetries; 40.22 + } 40.23 + if (retries_503) 40.24 + { 40.25 + *retries_503 = m503Retries; 40.26 + } 40.27 + } 40.28 + 40.29 + void setRetries(unsigned int retries, unsigned int retries_503) 40.30 + { 40.31 + mRetries = retries; 40.32 + m503Retries = retries_503; 40.33 + } 40.34 + 40.35 protected: 40.36 // Response data here 40.37 HttpStatus mStatus; 40.38 @@ -158,6 +177,8 @@ 40.39 BufferArray * mBufferArray; 40.40 HttpHeaders * mHeaders; 40.41 std::string mContentType; 40.42 + unsigned int mRetries; 40.43 + unsigned int m503Retries; 40.44 }; 40.45 40.46
41.1 --- a/indra/llcorehttp/tests/test_httprequest.hpp Mon Feb 24 11:33:41 2014 -0500 41.2 +++ b/indra/llcorehttp/tests/test_httprequest.hpp Tue Feb 25 13:25:40 2014 -0500 41.3 @@ -1222,7 +1222,7 @@ 41.4 HttpRequest::createService(); 41.5 41.6 // Enable tracing 41.7 - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); 41.8 + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); 41.9 41.10 // Start threading early so that thread memory is invariant 41.11 // over the test. 41.12 @@ -1340,7 +1340,7 @@ 41.13 HttpRequest::createService(); 41.14 41.15 // Enable tracing 41.16 - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); 41.17 + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); 41.18 41.19 // Start threading early so that thread memory is invariant 41.20 // over the test. 41.21 @@ -3175,6 +3175,142 @@ 41.22 } 41.23 } 41.24 41.25 +template <> template <> 41.26 +void HttpRequestTestObjectType::test<23>() 41.27 +{ 41.28 + ScopedCurlInit ready; 41.29 + 41.30 + set_test_name("HttpRequest GET 503s with 'Retry-After'"); 41.31 + 41.32 + // This tests mainly that the code doesn't fall over if 41.33 + // various well- and mis-formed Retry-After headers are 41.34 + // sent along with the response. Direct inspection of 41.35 + // the parsing result isn't supported. 41.36 + 41.37 + // Handler can be stack-allocated *if* there are no dangling 41.38 + // references to it after completion of this method. 41.39 + // Create before memory record as the string copy will bump numbers. 41.40 + TestHandler2 handler(this, "handler"); 41.41 + std::string url_base(get_base_url() + "/503/"); // path to 503 generators 41.42 + 41.43 + // record the total amount of dynamically allocated memory 41.44 + mMemTotal = GetMemTotal(); 41.45 + mHandlerCalls = 0; 41.46 + 41.47 + HttpRequest * req = NULL; 41.48 + HttpOptions * opts = NULL; 41.49 + 41.50 + try 41.51 + { 41.52 + // Get singletons created 41.53 + HttpRequest::createService(); 41.54 + 41.55 + // Start threading early so that thread memory is invariant 41.56 + // over the test. 41.57 + HttpRequest::startThread(); 41.58 + 41.59 + // create a new ref counted object with an implicit reference 41.60 + req = new HttpRequest(); 41.61 + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); 41.62 + 41.63 + opts = new HttpOptions(); 41.64 + opts->setRetries(1); // Retry once only 41.65 + opts->setUseRetryAfter(true); // Try to parse the retry-after header 41.66 + 41.67 + // Issue a GET that 503s with valid retry-after 41.68 + mStatus = HttpStatus(503); 41.69 + int url_limit(6); 41.70 + for (int i(0); i < url_limit; ++i) 41.71 + { 41.72 + std::ostringstream url; 41.73 + url << url_base << i << "/"; 41.74 + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 41.75 + 0U, 41.76 + url.str(), 41.77 + 0, 41.78 + 0, 41.79 + opts, 41.80 + NULL, 41.81 + &handler); 41.82 + 41.83 + std::ostringstream testtag; 41.84 + testtag << "Valid handle returned for 503 request #" << i; 41.85 + ensure(testtag.str(), handle != LLCORE_HTTP_HANDLE_INVALID); 41.86 + } 41.87 + 41.88 + 41.89 + // Run the notification pump. 41.90 + int count(0); 41.91 + int limit(LOOP_COUNT_LONG); 41.92 + while (count++ < limit && mHandlerCalls < url_limit) 41.93 + { 41.94 + req->update(0); 41.95 + usleep(LOOP_SLEEP_INTERVAL); 41.96 + } 41.97 + ensure("Request executed in reasonable time", count < limit); 41.98 + ensure("One handler invocation for request", mHandlerCalls == url_limit); 41.99 + 41.100 + // Okay, request a shutdown of the servicing thread 41.101 + mStatus = HttpStatus(); 41.102 + mHandlerCalls = 0; 41.103 + HttpHandle handle = req->requestStopThread(&handler); 41.104 + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); 41.105 + 41.106 + // Run the notification pump again 41.107 + count = 0; 41.108 + limit = LOOP_COUNT_LONG; 41.109 + while (count++ < limit && mHandlerCalls < 1) 41.110 + { 41.111 + req->update(1000000); 41.112 + usleep(LOOP_SLEEP_INTERVAL); 41.113 + } 41.114 + ensure("Second request executed in reasonable time", count < limit); 41.115 + ensure("Second handler invocation", mHandlerCalls == 1); 41.116 + 41.117 + // See that we actually shutdown the thread 41.118 + count = 0; 41.119 + limit = LOOP_COUNT_SHORT; 41.120 + while (count++ < limit && ! HttpService::isStopped()) 41.121 + { 41.122 + usleep(LOOP_SLEEP_INTERVAL); 41.123 + } 41.124 + ensure("Thread actually stopped running", HttpService::isStopped()); 41.125 + 41.126 + // release options 41.127 + opts->release(); 41.128 + opts = NULL; 41.129 + 41.130 + // release the request object 41.131 + delete req; 41.132 + req = NULL; 41.133 + 41.134 + // Shut down service 41.135 + HttpRequest::destroyService(); 41.136 + 41.137 +#if defined(WIN32) 41.138 + // Can only do this memory test on Windows. On other platforms, 41.139 + // the LL logging system holds on to memory and produces what looks 41.140 + // like memory leaks... 41.141 + 41.142 + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); 41.143 + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); 41.144 +#endif 41.145 + } 41.146 + catch (...) 41.147 + { 41.148 + stop_thread(req); 41.149 + if (opts) 41.150 + { 41.151 + opts->release(); 41.152 + opts = NULL; 41.153 + } 41.154 + delete req; 41.155 + HttpRequest::destroyService(); 41.156 + throw; 41.157 + } 41.158 +} 41.159 + 41.160 + 41.161 } // end namespace tut 41.162 41.163 namespace
42.1 --- a/indra/llcorehttp/tests/test_httpstatus.hpp Mon Feb 24 11:33:41 2014 -0500 42.2 +++ b/indra/llcorehttp/tests/test_httpstatus.hpp Tue Feb 25 13:25:40 2014 -0500 42.3 @@ -259,6 +259,65 @@ 42.4 ensure(msg == "Unknown error"); 42.5 } 42.6 42.7 + 42.8 +template <> template <> 42.9 +void HttpStatusTestObjectType::test<8>() 42.10 +{ 42.11 + set_test_name("HttpStatus toHex() nominal function"); 42.12 + 42.13 + HttpStatus status(404); 42.14 + std::string msg = status.toHex(); 42.15 + // std::cout << "Result: " << msg << std::endl; 42.16 + ensure(msg == "01940001"); 42.17 +} 42.18 + 42.19 + 42.20 +template <> template <> 42.21 +void HttpStatusTestObjectType::test<9>() 42.22 +{ 42.23 + set_test_name("HttpStatus toTerseString() nominal function"); 42.24 + 42.25 + HttpStatus status(404); 42.26 + std::string msg = status.toTerseString(); 42.27 + // std::cout << "Result: " << msg << std::endl; 42.28 + ensure("Normal HTTP 404", msg == "Http_404"); 42.29 + 42.30 + status = HttpStatus(200); 42.31 + msg = status.toTerseString(); 42.32 + // std::cout << "Result: " << msg << std::endl; 42.33 + ensure("Normal HTTP 200", msg == "Http_200"); 42.34 + 42.35 + status = HttpStatus(200, HE_REPLY_ERROR); 42.36 + msg = status.toTerseString(); 42.37 + // std::cout << "Result: " << msg << std::endl; 42.38 + ensure("Unsuccessful HTTP 200", msg == "Http_200"); // No distinction for error 42.39 + 42.40 + status = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); 42.41 + msg = status.toTerseString(); 42.42 + // std::cout << "Result: " << msg << std::endl; 42.43 + ensure("Easy couldn't connect error", msg == "Easy_7"); 42.44 + 42.45 + status = HttpStatus(HttpStatus::EXT_CURL_MULTI, CURLM_OUT_OF_MEMORY); 42.46 + msg = status.toTerseString(); 42.47 + // std::cout << "Result: " << msg << std::endl; 42.48 + ensure("Multi out-of-memory error", msg == "Multi_3"); 42.49 + 42.50 + status = HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_SET); 42.51 + msg = status.toTerseString(); 42.52 + // std::cout << "Result: " << msg << std::endl; 42.53 + ensure("Core option not set error", msg == "Core_7"); 42.54 + 42.55 + status = HttpStatus(22000, 1); 42.56 + msg = status.toTerseString(); 42.57 + // std::cout << "Result: " << msg << std::endl; 42.58 + ensure("Undecodable error", msg == "Unknown_1"); 42.59 + 42.60 + status = HttpStatus(22000, -1); 42.61 + msg = status.toTerseString(); 42.62 + // std::cout << "Result: " << msg << std::endl; 42.63 + ensure("Undecodable error 65535", msg == "Unknown_65535"); 42.64 +} 42.65 + 42.66 } // end namespace tut 42.67 42.68 #endif // TEST_HTTP_STATUS_H
43.1 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py Mon Feb 24 11:33:41 2014 -0500 43.2 +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py Tue Feb 25 13:25:40 2014 -0500 43.3 @@ -69,6 +69,15 @@ 43.4 "Content-Range: bytes 0-75/2983", 43.5 "Content-Length: 76" 43.6 -- '/bug2295/inv_cont_range/0/' Generates HE_INVALID_CONTENT_RANGE error in llcorehttp. 43.7 + - '/503/' Generate 503 responses with various kinds 43.8 + of 'retry-after' headers 43.9 + -- '/503/0/' "Retry-After: 2" 43.10 + -- '/503/1/' "Retry-After: Thu, 31 Dec 2043 23:59:59 GMT" 43.11 + -- '/503/2/' "Retry-After: Fri, 31 Dec 1999 23:59:59 GMT" 43.12 + -- '/503/3/' "Retry-After: " 43.13 + -- '/503/4/' "Retry-After: (*#*(@*(@(")" 43.14 + -- '/503/5/' "Retry-After: aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo" 43.15 + -- '/503/6/' "Retry-After: 1 2 3 4 5 6 7 8 9 10" 43.16 43.17 Some combinations make no sense, there's no effort to protect 43.18 you from that. 43.19 @@ -143,22 +152,40 @@ 43.20 if "/sleep/" in self.path: 43.21 time.sleep(30) 43.22 43.23 - if "fail" in self.path: 43.24 - status = data.get("status", 500) 43.25 - # self.responses maps an int status to a (short, long) pair of 43.26 - # strings. We want the longer string. That's why we pass a string 43.27 - # pair to get(): the [1] will select the second string, whether it 43.28 - # came from self.responses or from our default pair. 43.29 - reason = data.get("reason", 43.30 - self.responses.get(status, 43.31 - ("fail requested", 43.32 - "Your request specified failure status %s " 43.33 - "without providing a reason" % status))[1]) 43.34 - debug("fail requested: %s: %r", status, reason) 43.35 - self.send_error(status, reason) 43.36 + if "/503/" in self.path: 43.37 + # Tests for various kinds of 'Retry-After' header parsing 43.38 + body = None 43.39 + if "/503/0/" in self.path: 43.40 + self.send_response(503) 43.41 + self.send_header("retry-after", "2") 43.42 + elif "/503/1/" in self.path: 43.43 + self.send_response(503) 43.44 + self.send_header("retry-after", "Thu, 31 Dec 2043 23:59:59 GMT") 43.45 + elif "/503/2/" in self.path: 43.46 + self.send_response(503) 43.47 + self.send_header("retry-after", "Fri, 31 Dec 1999 23:59:59 GMT") 43.48 + elif "/503/3/" in self.path: 43.49 + self.send_response(503) 43.50 + self.send_header("retry-after", "") 43.51 + elif "/503/4/" in self.path: 43.52 + self.send_response(503) 43.53 + self.send_header("retry-after", "(*#*(@*(@(") 43.54 + elif "/503/5/" in self.path: 43.55 + self.send_response(503) 43.56 + self.send_header("retry-after", "aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo") 43.57 + elif "/503/6/" in self.path: 43.58 + self.send_response(503) 43.59 + self.send_header("retry-after", "1 2 3 4 5 6 7 8 9 10") 43.60 + else: 43.61 + # Unknown request 43.62 + self.send_response(400) 43.63 + body = "Unknown /503/ path in server" 43.64 if "/reflect/" in self.path: 43.65 self.reflect_headers() 43.66 + self.send_header("Content-type", "text/plain") 43.67 self.end_headers() 43.68 + if body: 43.69 + self.wfile.write(body) 43.70 elif "/bug2295/" in self.path: 43.71 # Test for https://jira.secondlife.com/browse/BUG-2295 43.72 # 43.73 @@ -194,8 +221,7 @@ 43.74 self.end_headers() 43.75 if body: 43.76 self.wfile.write(body) 43.77 - else: 43.78 - # Normal response path 43.79 + elif "fail" not in self.path: 43.80 data = data.copy() # we're going to modify 43.81 # Ensure there's a "reply" key in data, even if there wasn't before 43.82 data["reply"] = data.get("reply", llsd.LLSD("success")) 43.83 @@ -210,6 +236,22 @@ 43.84 self.end_headers() 43.85 if withdata: 43.86 self.wfile.write(response) 43.87 + else: # fail requested 43.88 + status = data.get("status", 500) 43.89 + # self.responses maps an int status to a (short, long) pair of 43.90 + # strings. We want the longer string. That's why we pass a string 43.91 + # pair to get(): the [1] will select the second string, whether it 43.92 + # came from self.responses or from our default pair. 43.93 + reason = data.get("reason", 43.94 + self.responses.get(status, 43.95 + ("fail requested", 43.96 + "Your request specified failure status %s " 43.97 + "without providing a reason" % status))[1]) 43.98 + debug("fail requested: %s: %r", status, reason) 43.99 + self.send_error(status, reason) 43.100 + if "/reflect/" in self.path: 43.101 + self.reflect_headers() 43.102 + self.end_headers() 43.103 43.104 def reflect_headers(self): 43.105 for name in self.headers.keys():
44.1 --- a/indra/llmessage/llcurl.cpp Mon Feb 24 11:33:41 2014 -0500 44.2 +++ b/indra/llmessage/llcurl.cpp Tue Feb 25 13:25:40 2014 -0500 44.3 @@ -6,7 +6,7 @@ 44.4 * 44.5 * $LicenseInfo:firstyear=2006&license=viewerlgpl$ 44.6 * Second Life Viewer Source Code 44.7 - * Copyright (C) 2010, Linden Research, Inc. 44.8 + * Copyright (C) 2010-2013, Linden Research, Inc. 44.9 * 44.10 * This library is free software; you can redistribute it and/or 44.11 * modify it under the terms of the GNU Lesser General Public 44.12 @@ -370,9 +370,12 @@ 44.13 return NULL; 44.14 } 44.15 44.16 - // set no DNS caching as default for all easy handles. This prevents them adopting a 44.17 - // multi handles cache if they are added to one. 44.18 - CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); 44.19 + // Enable a brief cache period for now. This was zero for the longest time 44.20 + // which caused some routers grief and generated unneeded traffic. For the 44.21 + // threaded resolver, we're using system resolution libraries and non-zero values 44.22 + // are preferred. The c-ares resolver is another matter and it might not 44.23 + // track server changes as well. 44.24 + CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); 44.25 check_curl_code(result); 44.26 result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); 44.27 check_curl_code(result);
45.1 --- a/indra/newview/VIEWER_VERSION.txt Mon Feb 24 11:33:41 2014 -0500 45.2 +++ b/indra/newview/VIEWER_VERSION.txt Tue Feb 25 13:25:40 2014 -0500 45.3 @@ -1,1 +1,1 @@ 45.4 -3.7.2 45.5 +3.7.3
46.1 --- a/indra/newview/app_settings/settings.xml Mon Feb 24 11:33:41 2014 -0500 46.2 +++ b/indra/newview/app_settings/settings.xml Tue Feb 25 13:25:40 2014 -0500 46.3 @@ -10068,11 +10068,21 @@ 46.4 <key>Value</key> 46.5 <real>16</real> 46.6 </map> 46.7 - 46.8 + <key>Mesh2MaxConcurrentRequests</key> 46.9 + <map> 46.10 + <key>Comment</key> 46.11 + <string>Number of connections to use for loading meshes.</string> 46.12 + <key>Persist</key> 46.13 + <integer>1</integer> 46.14 + <key>Type</key> 46.15 + <string>U32</string> 46.16 + <key>Value</key> 46.17 + <integer>8</integer> 46.18 + </map> 46.19 <key>MeshMaxConcurrentRequests</key> 46.20 <map> 46.21 <key>Comment</key> 46.22 - <string>Number of threads to use for loading meshes.</string> 46.23 + <string>Number of connections to use for loading meshes (legacy system).</string> 46.24 <key>Persist</key> 46.25 <integer>1</integer> 46.26 <key>Type</key> 46.27 @@ -10080,6 +10090,28 @@ 46.28 <key>Value</key> 46.29 <integer>32</integer> 46.30 </map> 46.31 + <key>MeshUseHttpRetryAfter</key> 46.32 + <map> 46.33 + <key>Comment</key> 46.34 + <string>If TRUE, use Retry-After response headers when rescheduling a mesh request that fails with an HTTP 503 status. Static.</string> 46.35 + <key>Persist</key> 46.36 + <integer>1</integer> 46.37 + <key>Type</key> 46.38 + <string>Boolean</string> 46.39 + <key>Value</key> 46.40 + <boolean>1</boolean> 46.41 + </map> 46.42 + <key>MeshUseGetMesh1</key> 46.43 + <map> 46.44 + <key>Comment</key> 46.45 + <string>If TRUE, use the legacy GetMesh capability for mesh download requests. Semi-dynamic (read at region crossings).</string> 46.46 + <key>Persist</key> 46.47 + <integer>1</integer> 46.48 + <key>Type</key> 46.49 + <string>Boolean</string> 46.50 + <key>Value</key> 46.51 + <boolean>0</boolean> 46.52 + </map> 46.53 <key>RunMultipleThreads</key> 46.54 <map> 46.55 <key>Comment</key>
47.1 --- a/indra/newview/llappcorehttp.cpp Mon Feb 24 11:33:41 2014 -0500 47.2 +++ b/indra/newview/llappcorehttp.cpp Tue Feb 25 13:25:40 2014 -0500 47.3 @@ -4,7 +4,7 @@ 47.4 * 47.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 47.6 * Second Life Viewer Source Code 47.7 - * Copyright (C) 2012, Linden Research, Inc. 47.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 47.9 * 47.10 * This library is free software; you can redistribute it and/or 47.11 * modify it under the terms of the GNU Lesser General Public 47.12 @@ -28,18 +28,81 @@ 47.13 47.14 #include "llappcorehttp.h" 47.15 47.16 +#include "llappviewer.h" 47.17 #include "llviewercontrol.h" 47.18 47.19 47.20 +// Here is where we begin to get our connection usage under control. 47.21 +// This establishes llcorehttp policy classes that, among other 47.22 +// things, limit the maximum number of connections to outside 47.23 +// services. Each of the entries below maps to a policy class and 47.24 +// has a limit, sometimes configurable, of how many connections can 47.25 +// be open at a time. 47.26 + 47.27 const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); 47.28 +static const struct 47.29 +{ 47.30 + LLAppCoreHttp::EAppPolicy mPolicy; 47.31 + U32 mDefault; 47.32 + U32 mMin; 47.33 + U32 mMax; 47.34 + U32 mRate; 47.35 + std::string mKey; 47.36 + const char * mUsage; 47.37 +} init_data[] = // Default and dynamic values for classes 47.38 +{ 47.39 + { 47.40 + LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, 47.41 + "", 47.42 + "other" 47.43 + }, 47.44 + { 47.45 + LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, 47.46 + "TextureFetchConcurrency", 47.47 + "texture fetch" 47.48 + }, 47.49 + { 47.50 + LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, 47.51 + "MeshMaxConcurrentRequests", 47.52 + "mesh fetch" 47.53 + }, 47.54 + { 47.55 + LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, 47.56 + "Mesh2MaxConcurrentRequests", 47.57 + "mesh2 fetch" 47.58 + }, 47.59 + { 47.60 + LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, 47.61 + "", 47.62 + "large mesh fetch" 47.63 + }, 47.64 + { 47.65 + LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, 47.66 + "", 47.67 + "asset upload" 47.68 + }, 47.69 + { 47.70 + LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, 47.71 + "", 47.72 + "long poll" 47.73 + } 47.74 +}; 47.75 + 47.76 +static void setting_changed(); 47.77 + 47.78 47.79 LLAppCoreHttp::LLAppCoreHttp() 47.80 : mRequest(NULL), 47.81 mStopHandle(LLCORE_HTTP_HANDLE_INVALID), 47.82 mStopRequested(0.0), 47.83 - mStopped(false), 47.84 - mPolicyDefault(-1) 47.85 -{} 47.86 + mStopped(false) 47.87 +{ 47.88 + for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) 47.89 + { 47.90 + mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; 47.91 + mSettings[i] = 0U; 47.92 + } 47.93 +} 47.94 47.95 47.96 LLAppCoreHttp::~LLAppCoreHttp() 47.97 @@ -54,30 +117,28 @@ 47.98 LLCore::HttpStatus status = LLCore::HttpRequest::createService(); 47.99 if (! status) 47.100 { 47.101 - LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " 47.102 - << status.toString() 47.103 + LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " << status.toString() 47.104 << LL_ENDL; 47.105 } 47.106 47.107 // Point to our certs or SSH/https: will fail on connect 47.108 - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, 47.109 - gDirUtilp->getCAFile()); 47.110 + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, 47.111 + LLCore::HttpRequest::GLOBAL_POLICY_ID, 47.112 + gDirUtilp->getCAFile(), NULL); 47.113 if (! status) 47.114 { 47.115 - LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " 47.116 - << status.toString() 47.117 + LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " << status.toString() 47.118 << LL_ENDL; 47.119 } 47.120 47.121 - // Establish HTTP Proxy. "LLProxy" is a special string which directs 47.122 - // the code to use LLProxy::applyProxySettings() to establish any 47.123 - // HTTP or SOCKS proxy for http operations. 47.124 - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); 47.125 + // Establish HTTP Proxy, if desired. 47.126 + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_LLPROXY, 47.127 + LLCore::HttpRequest::GLOBAL_POLICY_ID, 47.128 + 1, NULL); 47.129 if (! status) 47.130 { 47.131 - LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " 47.132 - << status.toString() 47.133 - << LL_ENDL; 47.134 + LL_WARNS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() 47.135 + << LL_ENDL; 47.136 } 47.137 47.138 // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): 47.139 @@ -90,47 +151,74 @@ 47.140 { 47.141 long trace_level(0L); 47.142 trace_level = long(gSavedSettings.getU32(http_trace)); 47.143 - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, trace_level); 47.144 + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE, 47.145 + LLCore::HttpRequest::GLOBAL_POLICY_ID, 47.146 + trace_level, NULL); 47.147 } 47.148 47.149 // Setup default policy and constrain if directed to 47.150 - mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; 47.151 - static const std::string texture_concur("TextureFetchConcurrency"); 47.152 - if (gSavedSettings.controlExists(texture_concur)) 47.153 + mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; 47.154 + 47.155 + // Setup additional policies based on table and some special rules 47.156 + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) 47.157 { 47.158 - U32 concur(llmin(gSavedSettings.getU32(texture_concur), U32(12))); 47.159 + const EAppPolicy policy(init_data[i].mPolicy); 47.160 47.161 - if (concur > 0) 47.162 + if (AP_DEFAULT == policy) 47.163 { 47.164 - LLCore::HttpStatus status; 47.165 - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyDefault, 47.166 - LLCore::HttpRequest::CP_CONNECTION_LIMIT, 47.167 - concur); 47.168 - if (! status) 47.169 - { 47.170 - LL_WARNS("Init") << "Unable to set texture fetch concurrency. Reason: " 47.171 - << status.toString() 47.172 - << LL_ENDL; 47.173 - } 47.174 - else 47.175 - { 47.176 - LL_INFOS("Init") << "Application settings overriding default texture fetch concurrency. New value: " 47.177 - << concur 47.178 - << LL_ENDL; 47.179 - } 47.180 + // Pre-created 47.181 + continue; 47.182 + } 47.183 + 47.184 + mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); 47.185 + if (! mPolicies[policy]) 47.186 + { 47.187 + // Use default policy (but don't accidentally modify default) 47.188 + LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage 47.189 + << ". Using default policy." 47.190 + << LL_ENDL; 47.191 + mPolicies[policy] = mPolicies[AP_DEFAULT]; 47.192 + continue; 47.193 } 47.194 } 47.195 + 47.196 + // Need a request object to handle dynamic options before setting them 47.197 + mRequest = new LLCore::HttpRequest; 47.198 + 47.199 + // Apply initial settings 47.200 + refreshSettings(true); 47.201 47.202 // Kick the thread 47.203 status = LLCore::HttpRequest::startThread(); 47.204 if (! status) 47.205 { 47.206 - LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " 47.207 - << status.toString() 47.208 + LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " << status.toString() 47.209 << LL_ENDL; 47.210 } 47.211 47.212 - mRequest = new LLCore::HttpRequest; 47.213 + // Register signals for settings and state changes 47.214 + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) 47.215 + { 47.216 + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) 47.217 + { 47.218 + LLPointer<LLControlVariable> cntrl_ptr = gSavedSettings.getControl(init_data[i].mKey); 47.219 + if (cntrl_ptr.isNull()) 47.220 + { 47.221 + LL_WARNS("Init") << "Unable to set signal on global setting '" << init_data[i].mKey 47.222 + << "'" << LL_ENDL; 47.223 + } 47.224 + else 47.225 + { 47.226 + mSettingsSignal[i] = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); 47.227 + } 47.228 + } 47.229 + } 47.230 +} 47.231 + 47.232 + 47.233 +void setting_changed() 47.234 +{ 47.235 + LLAppViewer::instance()->getAppCoreHttp().refreshSettings(false); 47.236 } 47.237 47.238 47.239 @@ -173,6 +261,11 @@ 47.240 } 47.241 } 47.242 47.243 + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) 47.244 + { 47.245 + mSettingsSignal[i].disconnect(); 47.246 + } 47.247 + 47.248 delete mRequest; 47.249 mRequest = NULL; 47.250 47.251 @@ -185,6 +278,78 @@ 47.252 } 47.253 } 47.254 47.255 +void LLAppCoreHttp::refreshSettings(bool initial) 47.256 +{ 47.257 + LLCore::HttpStatus status; 47.258 + 47.259 + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) 47.260 + { 47.261 + const EAppPolicy policy(init_data[i].mPolicy); 47.262 + 47.263 + // Set any desired throttle 47.264 + if (initial && init_data[i].mRate) 47.265 + { 47.266 + // Init-time only, can use the static setters here 47.267 + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, 47.268 + mPolicies[policy], 47.269 + init_data[i].mRate, 47.270 + NULL); 47.271 + if (! status) 47.272 + { 47.273 + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage 47.274 + << " throttle rate. Reason: " << status.toString() 47.275 + << LL_ENDL; 47.276 + } 47.277 + } 47.278 + 47.279 + // Get target connection concurrency value 47.280 + U32 setting(init_data[i].mDefault); 47.281 + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) 47.282 + { 47.283 + U32 new_setting(gSavedSettings.getU32(init_data[i].mKey)); 47.284 + if (new_setting) 47.285 + { 47.286 + // Treat zero settings as an ask for default 47.287 + setting = llclamp(new_setting, init_data[i].mMin, init_data[i].mMax); 47.288 + } 47.289 + } 47.290 + 47.291 + if (! initial && setting == mSettings[policy]) 47.292 + { 47.293 + // Unchanged, try next setting 47.294 + continue; 47.295 + } 47.296 + 47.297 + // Set it and report 47.298 + // *TODO: These are intended to be per-host limits when we can 47.299 + // support that in llcorehttp/libcurl. 47.300 + LLCore::HttpHandle handle; 47.301 + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, 47.302 + mPolicies[policy], 47.303 + setting, NULL); 47.304 + if (LLCORE_HTTP_HANDLE_INVALID == handle) 47.305 + { 47.306 + status = mRequest->getStatus(); 47.307 + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage 47.308 + << " concurrency. Reason: " << status.toString() 47.309 + << LL_ENDL; 47.310 + } 47.311 + else 47.312 + { 47.313 + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage 47.314 + << " concurrency. New value: " << setting 47.315 + << LL_ENDL; 47.316 + mSettings[policy] = setting; 47.317 + if (initial && setting != init_data[i].mDefault) 47.318 + { 47.319 + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage 47.320 + << " concurrency. New value: " << setting 47.321 + << LL_ENDL; 47.322 + } 47.323 + } 47.324 + } 47.325 +} 47.326 + 47.327 47.328 void LLAppCoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) 47.329 {
48.1 --- a/indra/newview/llappcorehttp.h Mon Feb 24 11:33:41 2014 -0500 48.2 +++ b/indra/newview/llappcorehttp.h Tue Feb 25 13:25:40 2014 -0500 48.3 @@ -4,7 +4,7 @@ 48.4 * 48.5 * $LicenseInfo:firstyear=2012&license=viewerlgpl$ 48.6 * Second Life Viewer Source Code 48.7 - * Copyright (C) 2012, Linden Research, Inc. 48.8 + * Copyright (C) 2012-2013, Linden Research, Inc. 48.9 * 48.10 * This library is free software; you can redistribute it and/or 48.11 * modify it under the terms of the GNU Lesser General Public 48.12 @@ -41,6 +41,117 @@ 48.13 class LLAppCoreHttp : public LLCore::HttpHandler 48.14 { 48.15 public: 48.16 + typedef LLCore::HttpRequest::policy_t policy_t; 48.17 + 48.18 + enum EAppPolicy 48.19 + { 48.20 + /// Catchall policy class. Not used yet 48.21 + /// but will have a generous concurrency 48.22 + /// limit. Deep queueing possible by having 48.23 + /// a chatty HTTP user. 48.24 + /// 48.25 + /// Destination: anywhere 48.26 + /// Protocol: http: or https: 48.27 + /// Transfer size: KB-MB 48.28 + /// Long poll: no 48.29 + /// Concurrency: high 48.30 + /// Request rate: unknown 48.31 + /// Pipelined: no 48.32 + AP_DEFAULT, 48.33 + 48.34 + /// Texture fetching policy class. Used to 48.35 + /// download textures via capability or SSA 48.36 + /// baking service. Deep queueing of requests. 48.37 + /// Do not share. 48.38 + /// 48.39 + /// Destination: simhost:12046 & bake-texture:80 48.40 + /// Protocol: http: 48.41 + /// Transfer size: KB-MB 48.42 + /// Long poll: no 48.43 + /// Concurrency: high 48.44 + /// Request rate: high 48.45 + /// Pipelined: soon 48.46 + AP_TEXTURE, 48.47 + 48.48 + /// Legacy mesh fetching policy class. Used to 48.49 + /// download textures via 'GetMesh' capability. 48.50 + /// To be deprecated. Do not share. 48.51 + /// 48.52 + /// Destination: simhost:12046 48.53 + /// Protocol: http: 48.54 + /// Transfer size: KB-MB 48.55 + /// Long poll: no 48.56 + /// Concurrency: dangerously high 48.57 + /// Request rate: high 48.58 + /// Pipelined: no 48.59 + AP_MESH1, 48.60 + 48.61 + /// New mesh fetching policy class. Used to 48.62 + /// download textures via 'GetMesh2' capability. 48.63 + /// Used when fetch request (typically one LOD) 48.64 + /// is 'small', currently defined as 2MB. 48.65 + /// Very deeply queued. Do not share. 48.66 + /// 48.67 + /// Destination: simhost:12046 48.68 + /// Protocol: http: 48.69 + /// Transfer size: KB-MB 48.70 + /// Long poll: no 48.71 + /// Concurrency: high 48.72 + /// Request rate: high 48.73 + /// Pipelined: soon 48.74 + AP_MESH2, 48.75 + 48.76 + /// Large mesh fetching policy class. Used to 48.77 + /// download textures via 'GetMesh' or 'GetMesh2' 48.78 + /// capability. Used when fetch request 48.79 + /// is not small to avoid head-of-line problem 48.80 + /// when large requests block a sequence of small, 48.81 + /// fast requests. Can be shared with similar 48.82 + /// traffic that can wait for longish stalls 48.83 + /// (default timeout 600S). 48.84 + /// 48.85 + /// Destination: simhost:12046 48.86 + /// Protocol: http: 48.87 + /// Transfer size: MB 48.88 + /// Long poll: no 48.89 + /// Concurrency: low 48.90 + /// Request rate: low 48.91 + /// Pipelined: soon 48.92 + AP_LARGE_MESH, 48.93 + 48.94 + /// Asset upload policy class. Used to store 48.95 + /// assets (mesh only at the moment) via 48.96 + /// changeable URL. Responses may take some 48.97 + /// time (default timeout 240S). 48.98 + /// 48.99 + /// Destination: simhost:12043 48.100 + /// Protocol: https: 48.101 + /// Transfer size: KB-MB 48.102 + /// Long poll: no 48.103 + /// Concurrency: low 48.104 + /// Request rate: low 48.105 + /// Pipelined: no 48.106 + AP_UPLOADS, 48.107 + 48.108 + /// Long-poll-type HTTP requests. Not 48.109 + /// bound by a connection limit. Requests 48.110 + /// will typically hang around for a long 48.111 + /// time (~30S). Only shareable with other 48.112 + /// long-poll requests. 48.113 + /// 48.114 + /// Destination: simhost:12043 48.115 + /// Protocol: https: 48.116 + /// Transfer size: KB 48.117 + /// Long poll: yes 48.118 + /// Concurrency: unlimited but low in practice 48.119 + /// Request rate: low 48.120 + /// Pipelined: no 48.121 + AP_LONG_POLL, 48.122 + 48.123 + AP_COUNT // Must be last 48.124 + }; 48.125 + 48.126 +public: 48.127 LLAppCoreHttp(); 48.128 ~LLAppCoreHttp(); 48.129 48.130 @@ -65,21 +176,27 @@ 48.131 // Notification when the stop request is complete. 48.132 virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); 48.133 48.134 - // Retrieve the policy class for default operations. 48.135 - int getPolicyDefault() const 48.136 + // Retrieve a policy class identifier for desired 48.137 + // application function. 48.138 + policy_t getPolicy(EAppPolicy policy) const 48.139 { 48.140 - return mPolicyDefault; 48.141 + return mPolicies[policy]; 48.142 } 48.143 + 48.144 + // Apply initial or new settings from the environment. 48.145 + void refreshSettings(bool initial); 48.146 48.147 private: 48.148 static const F64 MAX_THREAD_WAIT_TIME; 48.149 48.150 private: 48.151 - LLCore::HttpRequest * mRequest; 48.152 + LLCore::HttpRequest * mRequest; // Request queue to issue shutdowns 48.153 LLCore::HttpHandle mStopHandle; 48.154 F64 mStopRequested; 48.155 bool mStopped; 48.156 - int mPolicyDefault; 48.157 + policy_t mPolicies[AP_COUNT]; // Policy class id for each connection set 48.158 + U32 mSettings[AP_COUNT]; 48.159 + boost::signals2::connection mSettingsSignal[AP_COUNT]; // Signals to global settings that affect us 48.160 }; 48.161 48.162
49.1 --- a/indra/newview/llmeshrepository.cpp Mon Feb 24 11:33:41 2014 -0500 49.2 +++ b/indra/newview/llmeshrepository.cpp Tue Feb 25 13:25:40 2014 -0500 49.3 @@ -5,7 +5,7 @@ 49.4 * 49.5 * $LicenseInfo:firstyear=2005&license=viewerlgpl$ 49.6 * Second Life Viewer Source Code 49.7 - * Copyright (C) 2010, Linden Research, Inc. 49.8 + * Copyright (C) 2010-2014, Linden Research, Inc. 49.9 * 49.10 * This library is free software; you can redistribute it and/or 49.11 * modify it under the terms of the GNU Lesser General Public 49.12 @@ -38,11 +38,13 @@ 49.13 #include "llcallbacklist.h" 49.14 #include "llcurl.h" 49.15 #include "lldatapacker.h" 49.16 +#include "lldeadmantimer.h" 49.17 #include "llfloatermodelpreview.h" 49.18 #include "llfloaterperms.h" 49.19 #include "lleconomy.h" 49.20 #include "llimagej2c.h" 49.21 #include "llhost.h" 49.22 +#include "llmath.h" 49.23 #include "llnotificationsutil.h" 49.24 #include "llsd.h" 49.25 #include "llsdutil_math.h" 49.26 @@ -52,6 +54,7 @@ 49.27 #include "llviewercontrol.h" 49.28 #include "llviewerinventory.h" 49.29 #include "llviewermenufile.h" 49.30 +#include "llviewermessage.h" 49.31 #include "llviewerobjectlist.h" 49.32 #include "llviewerregion.h" 49.33 #include "llviewertexturelist.h" 49.34 @@ -65,6 +68,9 @@ 49.35 #include "llfoldertype.h" 49.36 #include "llviewerparcelmgr.h" 49.37 #include "lluploadfloaterobservers.h" 49.38 +#include "bufferarray.h" 49.39 +#include "bufferstream.h" 49.40 +#include "llfasttimer.h" 49.41 49.42 #include "boost/lexical_cast.hpp" 49.43 49.44 @@ -72,11 +78,296 @@ 49.45 #include "netdb.h" 49.46 #endif 49.47 49.48 -#include <queue> 49.49 + 49.50 +// Purpose 49.51 +// 49.52 +// The purpose of this module is to provide access between the viewer 49.53 +// and the asset system as regards to mesh objects. 49.54 +// 49.55 +// * High-throughput download of mesh assets from servers while 49.56 +// following best industry practices for network profile. 49.57 +// * Reliable expensing and upload of new mesh assets. 49.58 +// * Recovery and retry from errors when appropriate. 49.59 +// * Decomposition of mesh assets for preview and uploads. 49.60 +// * And most important: all of the above without exposing the 49.61 +// main thread to stalls due to deep processing or thread 49.62 +// locking actions. In particular, the following operations 49.63 +// on LLMeshRepository are very averse to any stalls: 49.64 +// * loadMesh 49.65 +// * getMeshHeader (For structural details, see: 49.66 +// http://wiki.secondlife.com/wiki/Mesh/Mesh_Asset_Format) 49.67 +// * notifyLoadedMeshes 49.68 +// * getSkinInfo 49.69 +// 49.70 +// Threads 49.71 +// 49.72 +// main Main rendering thread, very sensitive to locking and other stalls 49.73 +// repo Overseeing worker thread associated with the LLMeshRepoThread class 49.74 +// decom Worker thread for mesh decomposition requests 49.75 +// core HTTP worker thread: does the work but doesn't intrude here 49.76 +// uploadN 0-N temporary mesh upload threads (0-1 in practice) 49.77 +// 49.78 +// Sequence of Operations 49.79 +// 49.80 +// What follows is a description of the retrieval of one LOD for 49.81 +// a new mesh object. Work is performed by a series of short, quick 49.82 +// actions distributed over a number of threads. Each is meant 49.83 +// to proceed without stalling and the whole forms a deep request 49.84 +// pipeline to achieve throughput. Ellipsis indicates a return 49.85 +// or break in processing which is resumed elsewhere. 49.86 +// 49.87 +// main thread repo thread (run() method) 49.88 +// 49.89 +// loadMesh() invoked to request LOD 49.90 +// append LODRequest to mPendingRequests 49.91 +// ... 49.92 +// other mesh requests may be made 49.93 +// ... 49.94 +// notifyLoadedMeshes() invoked to stage work 49.95 +// append HeaderRequest to mHeaderReqQ 49.96 +// ... 49.97 +// scan mHeaderReqQ 49.98 +// issue 4096-byte GET for header 49.99 +// ... 49.100 +// onCompleted() invoked for GET 49.101 +// data copied 49.102 +// headerReceived() invoked 49.103 +// LLSD parsed 49.104 +// mMeshHeader, mMeshHeaderSize updated 49.105 +// scan mPendingLOD for LOD request 49.106 +// push LODRequest to mLODReqQ 49.107 +// ... 49.108 +// scan mLODReqQ 49.109 +// fetchMeshLOD() invoked 49.110 +// issue Byte-Range GET for LOD 49.111 +// ... 49.112 +// onCompleted() invoked for GET 49.113 +// data copied 49.114 +// lodReceived() invoked 49.115 +// unpack data into LLVolume 49.116 +// append LoadedMesh to mLoadedQ 49.117 +// ... 49.118 +// notifyLoadedMeshes() invoked again 49.119 +// scan mLoadedQ 49.120 +// notifyMeshLoaded() for LOD 49.121 +// setMeshAssetLoaded() invoked for system volume 49.122 +// notifyMeshLoaded() invoked for each interested object 49.123 +// ... 49.124 +// 49.125 +// Mutexes 49.126 +// 49.127 +// LLMeshRepository::mMeshMutex 49.128 +// LLMeshRepoThread::mMutex 49.129 +// LLMeshRepoThread::mHeaderMutex 49.130 +// LLMeshRepoThread::mSignal (LLCondition) 49.131 +// LLPhysicsDecomp::mSignal (LLCondition) 49.132 +// LLPhysicsDecomp::mMutex 49.133 +// LLMeshUploadThread::mMutex 49.134 +// 49.135 +// Mutex Order Rules 49.136 +// 49.137 +// 1. LLMeshRepoThread::mMutex before LLMeshRepoThread::mHeaderMutex 49.138 +// 2. LLMeshRepository::mMeshMutex before LLMeshRepoThread::mMutex 49.139 +// (There are more rules, haven't been extracted.) 49.140 +// 49.141 +// Data Member Access/Locking 49.142 +// 49.143 +// Description of how shared access to static and instance data 49.144 +// members is performed. Each member is followed by the name of 49.145 +// the mutex, if any, covering the data and then a list of data 49.146 +// access models each of which is a triplet of the following form: 49.147 +// 49.148 +// {ro, wo, rw}.{main, repo, any}.{mutex, none} 49.149 +// Type of access: read-only, write-only, read-write. 49.150 +// Accessing thread or 'any' 49.151 +// Relevant mutex held during access (several may be held) or 'none' 49.152 +// 49.153 +// A careful eye will notice some unsafe operations. Many of these 49.154 +// have an alibi of some form. Several types of alibi are identified 49.155 +// and listed here: 49.156 +// 49.157 +// [0] No alibi. Probably unsafe. 49.158 +// [1] Single-writer, self-consistent readers. Old data must 49.159 +// be tolerated by any reader but data will come true eventually. 49.160 +// [2] Like [1] but provides a hint about thread state. These 49.161 +// may be unsafe. 49.162 +// [3] empty() check outside of lock. Can me made safish when 49.163 +// done in double-check lock style. But this depends on 49.164 +// std:: implementation and memory model. 49.165 +// [4] Appears to be covered by a mutex but doesn't need one. 49.166 +// [5] Read of a double-checked lock. 49.167 +// 49.168 +// So, in addition to documentation, take this as a to-do/review 49.169 +// list and see if you can improve things. For porters to non-x86 49.170 +// architectures, the weaker memory models will make these platforms 49.171 +// probabilistically more susceptible to hitting race conditions. 49.172 +// True here and in other multi-thread code such as texture fetching. 49.173 +// (Strong memory models make weak programmers. Weak memory models 49.174 +// make strong programmers. Ref: arm, ppc, mips, alpha) 49.175 +// 49.176 +// LLMeshRepository: 49.177 +// 49.178 +// sBytesReceived none rw.repo.none, ro.main.none [1] 49.179 +// sMeshRequestCount " 49.180 +// sHTTPRequestCount " 49.181 +// sHTTPLargeRequestCount " 49.182 +// sHTTPRetryCount " 49.183 +// sHTTPErrorCount " 49.184 +// sLODPending mMeshMutex [4] rw.main.mMeshMutex 49.185 +// sLODProcessing Repo::mMutex rw.any.Repo::mMutex 49.186 +// sCacheBytesRead none rw.repo.none, ro.main.none [1] 49.187 +// sCacheBytesWritten " 49.188 +// sCacheReads " 49.189 +// sCacheWrites " 49.190 +// mLoadingMeshes mMeshMutex [4] rw.main.none, rw.any.mMeshMutex 49.191 +// mSkinMap none rw.main.none 49.192 +// mDecompositionMap none rw.main.none 49.193 +// mPendingRequests mMeshMutex [4] rw.main.mMeshMutex 49.194 +// mLoadingSkins mMeshMutex [4] rw.main.mMeshMutex 49.195 +// mPendingSkinRequests mMeshMutex [4] rw.main.mMeshMutex 49.196 +// mLoadingDecompositions mMeshMutex [4] rw.main.mMeshMutex 49.197 +// mPendingDecompositionRequests mMeshMutex [4] rw.main.mMeshMutex 49.198 +// mLoadingPhysicsShapes mMeshMutex [4] rw.main.mMeshMutex 49.199 +// mPendingPhysicsShapeRequests mMeshMutex [4] rw.main.mMeshMutex 49.200 +// mUploads none rw.main.none (upload thread accessing objects) 49.201 +// mUploadWaitList none rw.main.none (upload thread accessing objects) 49.202 +// mInventoryQ mMeshMutex [4] rw.main.mMeshMutex, ro.main.none [5] 49.203 +// mUploadErrorQ mMeshMutex rw.main.mMeshMutex, rw.any.mMeshMutex 49.204 +// mGetMeshVersion none rw.main.none 49.205 +// 49.206 +// LLMeshRepoThread: 49.207 +// 49.208 +// sActiveHeaderRequests mMutex rw.any.mMutex, ro.repo.none [1] 49.209 +// sActiveLODRequests mMutex rw.any.mMutex, ro.repo.none [1] 49.210 +// sMaxConcurrentRequests mMutex wo.main.none, ro.repo.none, ro.main.mMutex 49.211 +// mMeshHeader mHeaderMutex rw.repo.mHeaderMutex, ro.main.mHeaderMutex, ro.main.none [0] 49.212 +// mMeshHeaderSize mHeaderMutex rw.repo.mHeaderMutex 49.213 +// mSkinRequests mMutex rw.repo.mMutex, ro.repo.none [5] 49.214 +// mSkinInfoQ mMutex rw.repo.mMutex, rw.main.mMutex [5] (was: [0]) 49.215 +// mDecompositionRequests mMutex rw.repo.mMutex, ro.repo.none [5] 49.216 +// mPhysicsShapeRequests mMutex rw.repo.mMutex, ro.repo.none [5] 49.217 +// mDecompositionQ mMutex rw.repo.mMutex, rw.main.mMutex [5] (was: [0]) 49.218 +// mHeaderReqQ mMutex ro.repo.none [5], rw.repo.mMutex, rw.any.mMutex 49.219 +// mLODReqQ mMutex ro.repo.none [5], rw.repo.mMutex, rw.any.mMutex 49.220 +// mUnavailableQ mMutex rw.repo.none [0], ro.main.none [5], rw.main.mMutex 49.221 +// mLoadedQ mMutex rw.repo.mMutex, ro.main.none [5], rw.main.mMutex 49.222 +// mPendingLOD mMutex rw.repo.mMutex, rw.any.mMutex 49.223 +// mGetMeshCapability mMutex rw.main.mMutex, ro.repo.mMutex (was: [0]) 49.224 +// mGetMesh2Capability mMutex rw.main.mMutex, ro.repo.mMutex (was: [0]) 49.225 +// mGetMeshVersion mMutex rw.main.mMutex, ro.repo.mMutex 49.226 +// mHttp* none rw.repo.none 49.227 +// 49.228 +// LLMeshUploadThread: 49.229 +// 49.230 +// mDiscarded mMutex rw.main.mMutex, ro.uploadN.none [1] 49.231 +// ... more ... 49.232 +// 49.233 +// QA/Development Testing 49.234 +// 49.235 +// Debug variable 'MeshUploadFakeErrors' takes a mask of bits that will 49.236 +// simulate an error on fee query or upload. Defined bits are: 49.237 +// 49.238 +// 0x01 Simulate application error on fee check reading 49.239 +// response body from file "fake_upload_error.xml" 49.240 +// 0x02 Same as 0x01 but for actual upload attempt. 49.241 +// 0x04 Simulate a transport problem on fee check with a 49.242 +// locally-generated 500 status. 49.243 +// 0x08 As with 0x04 but for the upload operation. 49.244 +// 49.245 +// For major changes, see the LL_MESH_FASTTIMER_ENABLE below and 49.246 +// instructions for looking for frame stalls using fast timers. 49.247 +// 49.248 +// *TODO: Work list for followup actions: 49.249 +// * Review anything marked as unsafe above, verify if there are real issues. 49.250 +// * See if we can put ::run() into a hard sleep. May not actually perform better 49.251 +// than the current scheme so be prepared for disappointment. You'll likely 49.252 +// need to introduce a condition variable class that references a mutex in 49.253 +// methods rather than derives from mutex which isn't correct. 49.254 +// * On upload failures, make more information available to the alerting 49.255 +// dialog. Get the structured information going into the log into a 49.256 +// tree there. 49.257 +// * Header parse failures come without much explanation. Elaborate. 49.258 +// * Work queue for uploads? Any need for this or is the current scheme good 49.259 +// enough? 49.260 +// * Various temp buffers used in VFS I/O might be allocated once or even 49.261 +// statically. Look for some wins here. 49.262 +// * Move data structures holding mesh data used by main thread into main- 49.263 +// thread-only access so that no locking is needed. May require duplication 49.264 +// of some data so that worker thread has a minimal data set to guide 49.265 +// operations. 49.266 +// 49.267 +// -------------------------------------------------------------------------- 49.268 +// Development/Debug/QA Tools 49.269 +// 49.270 +// Enable here or in build environment to get fasttimer data on mesh fetches. 49.271 +// 49.272 +// Typically, this is used to perform A/B testing using the 49.273 +// fasttimer console (shift-ctrl-9). This is done by looking 49.274 +// for stalls due to lock contention between the main thread 49.275 +// and the repository and HTTP code. In a release viewer, 49.276 +// these appear as ping-time or worse spikes in frame time. 49.277 +// With this instrumentation enabled, a stall will appear 49.278 +// under the 'Mesh Fetch' timer which will be either top-level 49.279 +// or under 'Render' time. 49.280 + 49.281 +#ifndef LL_MESH_FASTTIMER_ENABLE 49.282 +#define LL_MESH_FASTTIMER_ENABLE 1 49.283 +#endif 49.284 +#if LL_MESH_FASTTIMER_ENABLE 49.285 +static LLFastTimer::DeclareTimer FTM_MESH_FETCH("Mesh Fetch"); 49.286 + 49.287 +#define MESH_FASTTIMER_DEFBLOCK LLFastTimer meshtimer(FTM_MESH_FETCH) 49.288 +#else 49.289 +#define MESH_FASTTIMER_DEFBLOCK 49.290 +#endif // LL_MESH_FASTTIMER_ENABLE 49.291 + 49.292 + 49.293 +// Random failure testing for development/QA. 49.294 +// 49.295 +// Set the MESH_*_FAILED macros to either 'false' or to 49.296 +// an invocation of MESH_RANDOM_NTH_TRUE() with some 49.297 +// suitable number. In production, all must be false. 49.298 +// 49.299 +// Example: 49.300 +// #define MESH_HTTP_RESPONSE_FAILED MESH_RANDOM_NTH_TRUE(9) 49.301 + 49.302 +// 1-in-N calls will test true 49.303 +#define MESH_RANDOM_NTH_TRUE(_N) ( ll_rand(S32(_N)) == 0 ) 49.304 + 49.305 +#define MESH_HTTP_RESPONSE_FAILED false 49.306 +#define MESH_HEADER_PROCESS_FAILED false 49.307 +#define MESH_LOD_PROCESS_FAILED false 49.308 +#define MESH_SKIN_INFO_PROCESS_FAILED false 49.309 +#define MESH_DECOMP_PROCESS_FAILED false 49.310 +#define MESH_PHYS_SHAPE_PROCESS_FAILED false 49.311 + 49.312 +// -------------------------------------------------------------------------- 49.313 + 49.314 49.315 LLMeshRepository gMeshRepo; 49.316 49.317 -const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; 49.318 +const S32 MESH_HEADER_SIZE = 4096; // Important: assumption is that headers fit in this space 49.319 +const S32 REQUEST_HIGH_WATER_MIN = 32; // Limits for GetMesh regions 49.320 +const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle 49.321 +const S32 REQUEST_LOW_WATER_MIN = 16; 49.322 +const S32 REQUEST_LOW_WATER_MAX = 75; 49.323 +const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions 49.324 +const S32 REQUEST2_HIGH_WATER_MAX = 80; 49.325 +const S32 REQUEST2_LOW_WATER_MIN = 16; 49.326 +const S32 REQUEST2_LOW_WATER_MAX = 40; 49.327 +const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue 49.328 +const long SMALL_MESH_XFER_TIMEOUT = 120L; // Seconds to complete xfer, small mesh downloads 49.329 +const long LARGE_MESH_XFER_TIMEOUT = 600L; // Seconds to complete xfer, large downloads 49.330 + 49.331 +// Would normally like to retry on uploads as some 49.332 +// retryable failures would be recoverable. Unfortunately, 49.333 +// the mesh service is using 500 (retryable) rather than 49.334 +// 400/bad request (permanent) for a bad payload and 49.335 +// retrying that just leads to revocation of the one-shot 49.336 +// cap which then produces a 404 on retry destroying some 49.337 +// (occasionally) useful error information. We'll leave 49.338 +// upload retries to the user as in the past. SH-4667. 49.339 +const long UPLOAD_RETRY_LIMIT = 0L; 49.340 49.341 // Maximum mesh version to support. Three least significant digits are reserved for the minor version, 49.342 // with major version changes indicating a format change that is not backwards compatible and should not 49.343 @@ -87,35 +378,45 @@ 49.344 const S32 MAX_MESH_VERSION = 999; 49.345 49.346 U32 LLMeshRepository::sBytesReceived = 0; 49.347 +U32 LLMeshRepository::sMeshRequestCount = 0; 49.348 U32 LLMeshRepository::sHTTPRequestCount = 0; 49.349 +U32 LLMeshRepository::sHTTPLargeRequestCount = 0; 49.350 U32 LLMeshRepository::sHTTPRetryCount = 0; 49.351 +U32 LLMeshRepository::sHTTPErrorCount = 0; 49.352 U32 LLMeshRepository::sLODProcessing = 0; 49.353 U32 LLMeshRepository::sLODPending = 0; 49.354 49.355 U32 LLMeshRepository::sCacheBytesRead = 0; 49.356 U32 LLMeshRepository::sCacheBytesWritten = 0; 49.357 -U32 LLMeshRepository::sPeakKbps = 0; 49.358 +U32 LLMeshRepository::sCacheReads = 0; 49.359 +U32 LLMeshRepository::sCacheWrites = 0; 49.360 +U32 LLMeshRepository::sMaxLockHoldoffs = 0; 49.361 + 49.362 +LLDeadmanTimer LLMeshRepository::sQuiescentTimer(15.0, false); // true -> gather cpu metrics 49.363 + 49.364 49.365 - 49.366 -const U32 MAX_TEXTURE_UPLOAD_RETRIES = 5; 49.367 - 49.368 static S32 dump_num = 0; 49.369 std::string make_dump_name(std::string prefix, S32 num) 49.370 { 49.371 return prefix + boost::lexical_cast<std::string>(num) + std::string(".xml"); 49.372 - 49.373 } 49.374 void dump_llsd_to_file(const LLSD& content, std::string filename); 49.375 LLSD llsd_from_file(std::string filename); 49.376 49.377 -std::string header_lod[] = 49.378 +const std::string header_lod[] = 49.379 { 49.380 "lowest_lod", 49.381 "low_lod", 49.382 "medium_lod", 49.383 "high_lod" 49.384 }; 49.385 - 49.386 +const char * const LOG_MESH = "Mesh"; 49.387 + 49.388 +// Static data and functions to measure mesh load 49.389 +// time metrics for a new region scene. 49.390 +static unsigned int metrics_teleport_start_count = 0; 49.391 +boost::signals2::connection metrics_teleport_started_signal; 49.392 +static void teleport_started(); 49.393 49.394 //get the number of bytes resident in memory for given volume 49.395 U32 get_volume_memory_size(const LLVolume* volume) 49.396 @@ -197,200 +498,233 @@ 49.397 } 49.398 } 49.399 49.400 -S32 LLMeshRepoThread::sActiveHeaderRequests = 0; 49.401 -S32 LLMeshRepoThread::sActiveLODRequests = 0; 49.402 +volatile S32 LLMeshRepoThread::sActiveHeaderRequests = 0; 49.403 +volatile S32 LLMeshRepoThread::sActiveLODRequests = 0; 49.404 U32 LLMeshRepoThread::sMaxConcurrentRequests = 1; 49.405 - 49.406 -class LLMeshHeaderResponder : public LLCurl::Responder 49.407 +S32 LLMeshRepoThread::sRequestLowWater = REQUEST2_LOW_WATER_MIN; 49.408 +S32 LLMeshRepoThread::sRequestHighWater = REQUEST2_HIGH_WATER_MIN; 49.409 +S32 LLMeshRepoThread::sRequestWaterLevel = 0; 49.410 + 49.411 +// Base handler class for all mesh users of llcorehttp. 49.412 +// This is roughly equivalent to a Responder class in 49.413 +// traditional LL code. The base is going to perform 49.414 +// common response/data handling in the inherited 49.415 +// onCompleted() method. Derived classes, one for each 49.416 +// type of HTTP action, define processData() and 49.417 +// processFailure() methods to customize handling and 49.418 +// error messages. 49.419 +// 49.420 +// LLCore::HttpHandler 49.421 +// LLMeshHandlerBase 49.422 +// LLMeshHeaderHandler 49.423 +// LLMeshLODHandler 49.424 +// LLMeshSkinInfoHandler 49.425 +// LLMeshDecompositionHandler 49.426 +// LLMeshPhysicsShapeHandler 49.427 +// LLMeshUploadThread 49.428 + 49.429 +class LLMeshHandlerBase : public LLCore::HttpHandler 49.430 { 49.431 - LOG_CLASS(LLMeshHeaderResponder); 49.432 public: 49.433 - LLVolumeParams mMeshParams; 49.434 - bool mProcessed; 49.435 - 49.436 - LLMeshHeaderResponder(const LLVolumeParams& mesh_params) 49.437 - : mMeshParams(mesh_params) 49.438 + LOG_CLASS(LLMeshHandlerBase); 49.439 + LLMeshHandlerBase() 49.440 + : LLCore::HttpHandler(), 49.441 + mMeshParams(), 49.442 + mProcessed(false), 49.443 + mHttpHandle(LLCORE_HTTP_HANDLE_INVALID) 49.444 + {} 49.445 + 49.446 + virtual ~LLMeshHandlerBase() 49.447 + {} 49.448 + 49.449 +protected: 49.450 + LLMeshHandlerBase(const LLMeshHandlerBase &); // Not defined 49.451 + void operator=(const LLMeshHandlerBase &); // Not defined 49.452 + 49.453 +public: 49.454 + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); 49.455 + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size) = 0; 49.456 + virtual void processFailure(LLCore::HttpStatus status) = 0; 49.457 + 49.458 +public: 49.459 + LLVolumeParams mMeshParams; 49.460 + bool mProcessed; 49.461 + LLCore::HttpHandle mHttpHandle; 49.462 +}; 49.463 + 49.464 + 49.465 +// Subclass for header fetches. 49.466 +// 49.467 +// Thread: repo 49.468 +class LLMeshHeaderHandler : public LLMeshHandlerBase 49.469 +{ 49.470 +public: 49.471 + LOG_CLASS(LLMeshHeaderHandler); 49.472 + LLMeshHeaderHandler(const LLVolumeParams & mesh_params) 49.473 + : LLMeshHandlerBase() 49.474 { 49.475 + mMeshParams = mesh_params; 49.476 LLMeshRepoThread::incActiveHeaderRequests(); 49.477 - mProcessed = false; 49.478 } 49.479 - 49.480 - ~LLMeshHeaderResponder() 49.481 + virtual ~LLMeshHeaderHandler(); 49.482 + 49.483 +protected: 49.484 + LLMeshHeaderHandler(const LLMeshHeaderHandler &); // Not defined 49.485 + void operator=(const LLMeshHeaderHandler &); // Not defined 49.486 + 49.487 +public: 49.488 + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); 49.489 + virtual void processFailure(LLCore::HttpStatus status); 49.490 +}; 49.491 + 49.492 + 49.493 +// Subclass for LOD fetches. 49.494 +// 49.495 +// Thread: repo 49.496 +class LLMeshLODHandler : public LLMeshHandlerBase 49.497 +{ 49.498 +public: 49.499 + LLMeshLODHandler(const LLVolumeParams & mesh_params, S32 lod, U32 offset, U32 requested_bytes) 49.500 + : LLMeshHandlerBase(), 49.501 + mLOD(lod), 49.502 + mRequestedBytes(requested_bytes), 49.503 + mOffset(offset) 49.504 { 49.505 - if (!LLApp::isQuitting()) 49.506 - { 49.507 - if (!mProcessed) 49.508 - { //something went wrong, retry 49.509 - llwarns << "Timeout or service unavailable, retrying." << llendl; 49.510 - LLMeshRepository::sHTTPRetryCount++; 49.511 - LLMeshRepoThread::HeaderRequest req(mMeshParams); 49.512 - LLMutexLock lock(gMeshRepo.mThread->mMutex); 49.513 - gMeshRepo.mThread->mHeaderReqQ.push(req); 49.514 - } 49.515 - 49.516 - LLMeshRepoThread::decActiveHeaderRequests(); 49.517 - } 49.518 + mMeshParams = mesh_params; 49.519 + LLMeshRepoThread::incActiveLODRequests(); 49.520 } 49.521 - 49.522 - virtual void completedRaw(const LLChannelDescriptors& channels, 49.523 - const LLIOPipe::buffer_ptr_t& buffer); 49.524 - 49.525 -}; 49.526 - 49.527 -class LLMeshLODResponder : public LLCurl::Responder 49.528 -{ 49.529 - LOG_CLASS(LLMeshLODResponder); 49.530 + virtual ~LLMeshLODHandler(); 49.531 + 49.532 +protected: 49.533 + LLMeshLODHandler(const LLMeshLODHandler &); // Not defined 49.534 + void operator=(const LLMeshLODHandler &); // Not defined 49.535 + 49.536 public: 49.537 - LLVolumeParams mMeshParams; 49.538 + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); 49.539 + virtual void processFailure(LLCore::HttpStatus status); 49.540 + 49.541 +public: 49.542 S32 mLOD; 49.543 U32 mRequestedBytes; 49.544 U32 mOffset; 49.545 - bool mProcessed; 49.546 - 49.547 - LLMeshLODResponder(const LLVolumeParams& mesh_params, S32 lod, U32 offset, U32 requested_bytes) 49.548 - : mMeshParams(mesh_params), mLOD(lod), mOffset(offset), mRequestedBytes(requested_bytes) 49.549 - { 49.550 - LLMeshRepoThread::incActiveLODRequests(); 49.551 - mProcessed = false; 49.552 - } 49.553 - 49.554 - ~LLMeshLODResponder() 49.555 - { 49.556 - if (!LLApp::isQuitting()) 49.557 - { 49.558 - if (!mProcessed) 49.559 - { 49.560 - llwarns << "Killed without being processed, retrying." << llendl; 49.561 - LLMeshRepository::sHTTPRetryCount++; 49.562 - gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD); 49.563 - } 49.564 - LLMeshRepoThread::decActiveLODRequests(); 49.565 - } 49.566 - } 49.567 - 49.568 - virtual void completedRaw(const LLChannelDescriptors& channels, 49.569 - const LLIOPipe::buffer_ptr_t& buffer); 49.570 - 49.571 }; 49.572 49.573 -class LLMeshSkinInfoResponder : public LLCurl::Responder 49.574 + 49.575 +// Subclass for skin info fetches. 49.576 +// 49.577 +// Thread: repo 49.578 +class LLMeshSkinInfoHandler : public LLMeshHandlerBase 49.579 { 49.580 - LOG_CLASS(LLMeshSkinInfoResponder); 49.581 +public: 49.582 + LOG_CLASS(LLMeshSkinInfoHandler); 49.583 + LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 size) 49.584 + : LLMeshHandlerBase(), 49.585 + mMeshID(id), 49.586 + mRequestedBytes(size), 49.587 + mOffset(offset) 49.588 + {} 49.589 + virtual ~LLMeshSkinInfoHandler(); 49.590 + 49.591 +protected: 49.592 + LLMeshSkinInfoHandler(const LLMeshSkinInfoHandler &); // Not defined 49.593 + void operator=(const LLMeshSkinInfoHandler &); // Not defined 49.594 + 49.595 +public: 49.596 + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); 49.597 + virtual void processFailure(LLCore::HttpStatus status); 49.598 + 49.599 public: 49.600 LLUUID mMeshID; 49.601 U32 mRequestedBytes; 49.602 U32 mOffset; 49.603 - bool mProcessed; 49.604 - 49.605 - LLMeshSkinInfoResponder(const LLUUID& id, U32 offset, U32 size) 49.606 - : mMeshID(id), mRequestedBytes(size), mOffset(offset) 49.607 - { 49.608 - mProcessed = false; 49.609 - } 49.610 - 49.611 - ~LLMeshSkinInfoResponder() 49.612 - { 49.613 - if (!LLApp::isQuitting() && 49.614 - !mProcessed && 49.615 - mMeshID.notNull()) 49.616 - { // Something went wrong, retry 49.617 - llwarns << "Timeout or service unavailable, retrying loadMeshSkinInfo() for " << mMeshID << llendl; 49.618 - LLMeshRepository::sHTTPRetryCount++; 49.619 - gMeshRepo.mThread->loadMeshSkinInfo(mMeshID); 49.620 - } 49.621 - } 49.622 - 49.623 - virtual void completedRaw(const LLChannelDescriptors& channels, 49.624 - const LLIOPipe::buffer_ptr_t& buffer); 49.625 - 49.626 }; 49.627 49.628 -class LLMeshDecompositionResponder : public LLCurl::Responder 49.629 + 49.630 +// Subclass for decomposition fetches. 49.631 +// 49.632 +// Thread: repo 49.633 +class LLMeshDecompositionHandler : public LLMeshHandlerBase 49.634 { 49.635 - LOG_CLASS(LLMeshDecompositionResponder); 49.636 +public: 49.637 + LOG_CLASS(LLMeshDecompositionHandler); 49.638 + LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 size) 49.639 + : LLMeshHandlerBase(), 49.640 + mMeshID(id), 49.641 + mRequestedBytes(size), 49.642 + mOffset(offset) 49.643 + {} 49.644 + virtual ~LLMeshDecompositionHandler(); 49.645 + 49.646 +protected: 49.647 + LLMeshDecompositionHandler(const LLMeshDecompositionHandler &); // Not defined 49.648 + void operator=(const LLMeshDecompositionHandler &); // Not defined 49.649 + 49.650 +public: 49.651 + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); 49.652 + virtual void processFailure(LLCore::HttpStatus status); 49.653 + 49.654 public: 49.655 LLUUID mMeshID; 49.656 U32 mRequestedBytes; 49.657 U32 mOffset; 49.658 - bool mProcessed; 49.659 - 49.660 - LLMeshDecompositionResponder(const LLUUID& id, U32 offset, U32 size) 49.661 - : mMeshID(id), mRequestedBytes(size), mOffset(offset) 49.662 - { 49.663 - mProcessed = false; 49.664 - } 49.665 - 49.666 - ~LLMeshDecompositionResponder() 49.667 - { 49.668 - if (!LLApp::isQuitting() && 49.669 - !mProcessed && 49.670 - mMeshID.notNull()) 49.671 - { // Something went wrong, retry 49.672 - llwarns << "Timeout or service unavailable, retrying loadMeshDecomposition() for " << mMeshID << llendl; 49.673 - LLMeshRepository::sHTTPRetryCount++; 49.674 - gMeshRepo.mThread->loadMeshDecomposition(mMeshID); 49.675 - } 49.676 - } 49.677 - 49.678 - virtual void completedRaw(const LLChannelDescriptors& channels, 49.679 - const LLIOPipe::buffer_ptr_t& buffer); 49.680 - 49.681 }; 49.682 49.683 -class LLMeshPhysicsShapeResponder : public LLCurl::Responder 49.684 + 49.685 +// Subclass for physics shape fetches. 49.686 +// 49.687 +// Thread: repo 49.688 +class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase 49.689 { 49.690 - LOG_CLASS(LLMeshPhysicsShapeResponder); 49.691 public: 49.692 - LLUUID mMeshID; 49.693 - U32 mRequestedBytes; 49.694 - U32 mOffset; 49.695 - bool mProcessed; 49.696 - 49.697 - LLMeshPhysicsShapeResponder(const LLUUID& id, U32 offset, U32 size) 49.698 - : mMeshID(id), mRequestedBytes(size), mOffset(offset) 49.699 - { 49.700 - mProcessed = false; 49.701 - } 49.702 - 49.703 - ~LLMeshPhysicsShapeResponder() 49.704 - { 49.705 - if (!LLApp::isQuitting() && 49.706 - !mProcessed && 49.707 - mMeshID.notNull()) 49.708 - { // Something went wrong, retry 49.709 - llwarns << "Timeout or service unavailable, retrying loadMeshPhysicsShape() for " << mMeshID << llendl; 49.710 - LLMeshRepository::sHTTPRetryCount++; 49.711 - gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID); 49.712 - } 49.713 - } 49.714 - 49.715 - virtual void completedRaw(const LLChannelDescriptors& channels, 49.716 - const LLIOPipe::buffer_ptr_t& buffer); 49.717 - 49.718 + LOG_CLASS(LLMeshPhysicsShapeHandler); 49.719 + LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 size) 49.720 + : LLMeshHandlerBase(), 49.721 + mMeshID(id), 49.722 + mRequestedBytes(size), 49.723 + mOffset(offset) 49.724 + {} 49.725 + virtual ~LLMeshPhysicsShapeHandler(); 49.726 + 49.727 +protected: 49.728 + LLMeshPhysicsShapeHandler(const LLMeshPhysicsShapeHandler &); // Not defined 49.729 + void operator=(const LLMeshPhysicsShapeHandler &); // Not defined 49.730 + 49.731 +public: 49.732 + virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); 49.733 + virtual void processFailure(LLCore::HttpStatus status); 49.734 + 49.735 +public: 49.736 + LLUUID mMeshID; 49.737 + U32 mRequestedBytes; 49.738 + U32 mOffset; 49.739 }; 49.740 49.741 -void log_upload_error(S32 status, const LLSD& content, std::string stage, std::string model_name) 49.742 + 49.743 +void log_upload_error(LLCore::HttpStatus status, const LLSD& content, 49.744 + const char * const stage, const std::string & model_name) 49.745 { 49.746 // Add notification popup. 49.747 LLSD args; 49.748 - std::string message = content["error"]["message"]; 49.749 - std::string identifier = content["error"]["identifier"]; 49.750 + std::string message = content["error"]["message"].asString(); 49.751 + std::string identifier = content["error"]["identifier"].asString(); 49.752 args["MESSAGE"] = message; 49.753 args["IDENTIFIER"] = identifier; 49.754 args["LABEL"] = model_name; 49.755 gMeshRepo.uploadError(args); 49.756 49.757 // Log details. 49.758 - llwarns << "stage: " << stage << " http status: " << status << llendl; 49.759 + LL_WARNS(LOG_MESH) << "Error in stage: " << stage 49.760 + << ", Reason: " << status.toString() 49.761 + << " (" << status.toTerseString() << ")" << LL_ENDL; 49.762 if (content.has("error")) 49.763 { 49.764 const LLSD& err = content["error"]; 49.765 - llwarns << "err: " << err << llendl; 49.766 - llwarns << "mesh upload failed, stage '" << stage 49.767 - << "' error '" << err["error"].asString() 49.768 - << "', message '" << err["message"].asString() 49.769 - << "', id '" << err["identifier"].asString() 49.770 - << "'" << llendl; 49.771 + LL_WARNS(LOG_MESH) << "error: " << err << LL_ENDL; 49.772 + LL_WARNS(LOG_MESH) << " mesh upload failed, stage '" << stage 49.773 + << "', error '" << err["error"].asString() 49.774 + << "', message '" << err["message"].asString() 49.775 + << "', id '" << err["identifier"].asString() 49.776 + << "'" << LL_ENDL; 49.777 if (err.has("errors")) 49.778 { 49.779 S32 error_num = 0; 49.780 @@ -400,13 +734,13 @@ 49.781 ++it) 49.782 { 49.783 const LLSD& err_entry = *it; 49.784 - llwarns << "error[" << error_num << "]:" << llendl; 49.785 + LL_WARNS(LOG_MESH) << " error[" << error_num << "]:" << LL_ENDL; 49.786 for (LLSD::map_const_iterator map_it = err_entry.beginMap(); 49.787 map_it != err_entry.endMap(); 49.788 ++map_it) 49.789 { 49.790 - llwarns << "\t" << map_it->first << ": " 49.791 - << map_it->second << llendl; 49.792 + LL_WARNS(LOG_MESH) << " " << map_it->first << ": " 49.793 + << map_it->second << LL_ENDL; 49.794 } 49.795 error_num++; 49.796 } 49.797 @@ -414,154 +748,71 @@ 49.798 } 49.799 else 49.800 { 49.801 - llwarns << "bad mesh, no error information available" << llendl; 49.802 + LL_WARNS(LOG_MESH) << "Bad response to mesh request, no additional error information available." << LL_ENDL; 49.803 } 49.804 } 49.805 49.806 -class LLWholeModelFeeResponder: public LLCurl::Responder 49.807 -{ 49.808 - LOG_CLASS(LLWholeModelFeeResponder); 49.809 - LLMeshUploadThread* mThread; 49.810 - LLSD mModelData; 49.811 - LLHandle<LLWholeModelFeeObserver> mObserverHandle; 49.812 -public: 49.813 - LLWholeModelFeeResponder(LLMeshUploadThread* thread, LLSD& model_data, LLHandle<LLWholeModelFeeObserver> observer_handle): 49.814 - mThread(thread), 49.815 - mModelData(model_data), 49.816 - mObserverHandle(observer_handle) 49.817 - { 49.818 - if (mThread) 49.819 - { 49.820 - mThread->startRequest(); 49.821 - } 49.822 - } 49.823 - 49.824 - ~LLWholeModelFeeResponder() 49.825 - { 49.826 - if (mThread) 49.827 - { 49.828 - mThread->stopRequest(); 49.829 - } 49.830 - } 49.831 - 49.832 -protected: 49.833 - virtual void httpCompleted() 49.834 - { 49.835 - LLSD cc = getContent(); 49.836 - if (gSavedSettings.getS32("MeshUploadFakeErrors")&1) 49.837 - { 49.838 - cc = llsd_from_file("fake_upload_error.xml"); 49.839 - } 49.840 - 49.841 - dump_llsd_to_file(cc,make_dump_name("whole_model_fee_response_",dump_num)); 49.842 - 49.843 - LLWholeModelFeeObserver* observer = mObserverHandle.get(); 49.844 - 49.845 - if (isGoodStatus() && 49.846 - cc["state"].asString() == "upload") 49.847 - { 49.848 - mThread->mWholeModelUploadURL = cc["uploader"].asString(); 49.849 - 49.850 - if (observer) 49.851 - { 49.852 - cc["data"]["upload_price"] = cc["upload_price"]; 49.853 - observer->onModelPhysicsFeeReceived(cc["data"], mThread->mWholeModelUploadURL); 49.854 - } 49.855 - } 49.856 - else 49.857 - { 49.858 - llwarns << "fee request failed " << dumpResponse() << llendl; 49.859 - S32 status = getStatus(); 49.860 - log_upload_error(status,cc,"fee",mModelData["name"]); 49.861 - mThread->mWholeModelUploadURL = ""; 49.862 - 49.863 - if (observer) 49.864 - { 49.865 - observer->setModelPhysicsFeeErrorStatus(status, getReason()); 49.866 - } 49.867 - } 49.868 - } 49.869 - 49.870 -}; 49.871 - 49.872 -class LLWholeModelUploadResponder: public LLCurl::Responder 49.873 -{ 49.874 - LOG_CLASS(LLWholeModelUploadResponder); 49.875 - LLMeshUploadThread* mThread; 49.876 - LLSD mModelData; 49.877 - LLHandle<LLWholeModelUploadObserver> mObserverHandle; 49.878 - 49.879 -public: 49.880 - LLWholeModelUploadResponder(LLMeshUploadThread* thread, LLSD& model_data, LLHandle<LLWholeModelUploadObserver> observer_handle): 49.881 - mThread(thread), 49.882 - mModelData(model_data), 49.883 - mObserverHandle(observer_handle) 49.884 - { 49.885 - if (mThread) 49.886 - { 49.887 - mThread->startRequest(); 49.888 - } 49.889 - } 49.890 - 49.891 - ~LLWholeModelUploadResponder() 49.892 - { 49.893 - if (mThread) 49.894 - { 49.895 - mThread->stopRequest(); 49.896 - } 49.897 - } 49.898 - 49.899 -protected: 49.900 - virtual void httpCompleted() 49.901 - { 49.902 - LLSD cc = getContent(); 49.903 - if (gSavedSettings.getS32("MeshUploadFakeErrors")&2) 49.904 - { 49.905 - cc = llsd_from_file("fake_upload_error.xml"); 49.906 - } 49.907 - 49.908 - dump_llsd_to_file(cc,make_dump_name("whole_model_upload_response_",dump_num)); 49.909 - 49.910 - LLWholeModelUploadObserver* observer = mObserverHandle.get(); 49.911 - 49.912 - // requested "mesh" asset type isn't actually the type 49.913 - // of the resultant object, fix it up here. 49.914 - if (isGoodStatus() && 49.915 - cc["state"].asString() == "complete") 49.916 - { 49.917 - mModelData["asset_type"] = "object"; 49.918 - gMeshRepo.updateInventory(LLMeshRepository::inventory_data(mModelData,cc)); 49.919 - 49.920 - if (observer) 49.921 - { 49.922 - doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadSuccess, observer)); 49.923 - } 49.924 - } 49.925 - else 49.926 - { 49.927 - llwarns << "upload failed " << dumpResponse() << llendl; 49.928 - std::string model_name = mModelData["name"].asString(); 49.929 - log_upload_error(getStatus(),cc,"upload",model_name); 49.930 - 49.931 - if (observer) 49.932 - { 49.933 - doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); 49.934 - } 49.935 - } 49.936 - } 49.937 -}; 49.938 - 49.939 LLMeshRepoThread::LLMeshRepoThread() 49.940 -: LLThread("mesh repo") 49.941 +: LLThread("mesh repo"), 49.942 + mHttpRequest(NULL), 49.943 + mHttpOptions(NULL), 49.944 + mHttpLargeOptions(NULL), 49.945 + mHttpHeaders(NULL), 49.946 + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), 49.947 + mHttpLegacyPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), 49.948 + mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), 49.949 + mHttpPriority(0), 49.950 + mGetMeshVersion(2) 49.951 { 49.952 - mWaiting = false; 49.953 mMutex = new LLMutex(NULL); 49.954 mHeaderMutex = new LLMutex(NULL); 49.955 mSignal = new LLCondition(NULL); 49.956 + mHttpRequest = new LLCore::HttpRequest; 49.957 + mHttpOptions = new LLCore::HttpOptions; 49.958 + mHttpOptions->setTransferTimeout(SMALL_MESH_XFER_TIMEOUT); 49.959 + mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); 49.960 + mHttpLargeOptions = new LLCore::HttpOptions; 49.961 + mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); 49.962 + mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); 49.963 + mHttpHeaders = new LLCore::HttpHeaders; 49.964 + mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); 49.965 + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH2); 49.966 + mHttpLegacyPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH1); 49.967 + mHttpLargePolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); 49.968 } 49.969 49.970 + 49.971 LLMeshRepoThread::~LLMeshRepoThread() 49.972 { 49.973 + LL_INFOS(LOG_MESH) << "Small GETs issued: " << LLMeshRepository::sHTTPRequestCount 49.974 + << ", Large GETs issued: " << LLMeshRepository::sHTTPLargeRequestCount 49.975 + << ", Max Lock Holdoffs: " << LLMeshRepository::sMaxLockHoldoffs 49.976 + << LL_ENDL; 49.977 + 49.978 + for (http_request_set::iterator iter(mHttpRequestSet.begin()); 49.979 + iter != mHttpRequestSet.end(); 49.980 + ++iter) 49.981 + { 49.982 + delete *iter; 49.983 + } 49.984 + mHttpRequestSet.clear(); 49.985 + if (mHttpHeaders) 49.986 + { 49.987 + mHttpHeaders->release(); 49.988 + mHttpHeaders = NULL; 49.989 + } 49.990 + if (mHttpOptions) 49.991 + { 49.992 + mHttpOptions->release(); 49.993 + mHttpOptions = NULL; 49.994 + } 49.995 + if (mHttpLargeOptions) 49.996 + { 49.997 + mHttpLargeOptions->release(); 49.998 + mHttpLargeOptions = NULL; 49.999 + } 49.1000 + delete mHttpRequest; 49.1001 + mHttpRequest = NULL; 49.1002 delete mMutex; 49.1003 mMutex = NULL; 49.1004 delete mHeaderMutex; 49.1005 @@ -572,109 +823,180 @@ 49.1006 49.1007 void LLMeshRepoThread::run() 49.1008 { 49.1009 - mCurlRequest = new LLCurlRequest(); 49.1010 LLCDResult res = LLConvexDecomposition::initThread(); 49.1011 if (res != LLCD_OK) 49.1012 { 49.1013 - llwarns << "convex decomposition unable to be loaded" << llendl; 49.1014 + LL_WARNS(LOG_MESH) << "Convex decomposition unable to be loaded. Expect severe problems." << LL_ENDL; 49.1015 } 49.1016 49.1017 while (!LLApp::isQuitting()) 49.1018 { 49.1019 - mWaiting = true; 49.1020 + // *TODO: Revise sleep/wake strategy and try to move away 49.1021 + // from polling operations in this thread. We can sleep 49.1022 + // this thread hard when: 49.1023 + // * All Http requests are serviced 49.1024 + // * LOD request queue empty 49.1025 + // * Header request queue empty 49.1026 + // * Skin info request queue empty 49.1027 + // * Decomposition request queue empty 49.1028 + // * Physics shape request queue empty 49.1029 + // We wake the thread when any of the above become untrue. 49.1030 + // Will likely need a correctly-implemented condition variable to do this. 49.1031 + // On the other hand, this may actually be an effective and efficient scheme... 49.1032 + 49.1033 mSignal->wait(); 49.1034 - mWaiting = false; 49.1035 - 49.1036 - if (!LLApp::isQuitting()) 49.1037 + 49.1038 + if (LLApp::isQuitting()) 49.1039 { 49.1040 - static U32 count = 0; 49.1041 - 49.1042 - static F32 last_hundred = gFrameTimeSeconds; 49.1043 - 49.1044 - if (gFrameTimeSeconds - last_hundred > 1.f) 49.1045 - { //a second has gone by, clear count 49.1046 - last_hundred = gFrameTimeSeconds; 49.1047 - count = 0; 49.1048 + break; 49.1049 + } 49.1050 + 49.1051 + if (! mHttpRequestSet.empty()) 49.1052 + { 49.1053 + // Dispatch all HttpHandler notifications 49.1054 + mHttpRequest->update(0L); 49.1055 + } 49.1056 + sRequestWaterLevel = mHttpRequestSet.size(); // Stats data update 49.1057 + 49.1058 + // NOTE: order of queue processing intentionally favors LOD requests over header requests 49.1059 + 49.1060 + while (!mLODReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) 49.1061 + { 49.1062 + if (! mMutex) 49.1063 + { 49.1064 + break; 49.1065 } 49.1066 - 49.1067 - // NOTE: throttling intentionally favors LOD requests over header requests 49.1068 - 49.1069 - while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveLODRequests < sMaxConcurrentRequests) 49.1070 + mMutex->lock(); 49.1071 + LODRequest req = mLODReqQ.front(); 49.1072 + mLODReqQ.pop(); 49.1073 + LLMeshRepository::sLODProcessing--; 49.1074 + mMutex->unlock(); 49.1075 + if (!fetchMeshLOD(req.mMeshParams, req.mLOD)) // failed, resubmit 49.1076 { 49.1077 - if (mMutex) 49.1078 - { 49.1079 - mMutex->lock(); 49.1080 - LODRequest req = mLODReqQ.front(); 49.1081 - mLODReqQ.pop(); 49.1082 - LLMeshRepository::sLODProcessing--; 49.1083 - mMutex->unlock(); 49.1084 - if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit 49.1085 - { 49.1086 - mMutex->lock(); 49.1087 - mLODReqQ.push(req); 49.1088 - mMutex->unlock(); 49.1089 - } 49.1090 - } 49.1091 + mMutex->lock(); 49.1092 + mLODReqQ.push(req) ; 49.1093 + ++LLMeshRepository::sLODProcessing; 49.1094 + mMutex->unlock(); 49.1095 } 49.1096 - 49.1097 - while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveHeaderRequests < sMaxConcurrentRequests) 49.1098 + } 49.1099 + 49.1100 + while (!mHeaderReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) 49.1101 + { 49.1102 + if (! mMutex) 49.1103 { 49.1104 - if (mMutex) 49.1105 - { 49.1106 - mMutex->lock(); 49.1107 - HeaderRequest req = mHeaderReqQ.front(); 49.1108 - mHeaderReqQ.pop(); 49.1109 - mMutex->unlock(); 49.1110 - if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit 49.1111 - { 49.1112 - mMutex->lock(); 49.1113 - mHeaderReqQ.push(req) ; 49.1114 - mMutex->unlock(); 49.1115 - } 49.1116 - } 49.1117 + break; 49.1118 } 49.1119 - 49.1120 - { //mSkinRequests is protected by mSignal 49.1121 + mMutex->lock(); 49.1122 + HeaderRequest req = mHeaderReqQ.front(); 49.1123 + mHeaderReqQ.pop(); 49.1124 + mMutex->unlock(); 49.1125 + if (!fetchMeshHeader(req.mMeshParams))//failed, resubmit 49.1126 + { 49.1127 + mMutex->lock(); 49.1128 + mHeaderReqQ.push(req) ; 49.1129 + mMutex->unlock(); 49.1130 + } 49.1131 + } 49.1132 + 49.1133 + // For the final three request lists, similar goal to above but 49.1134 + // slightly different queue structures. Stay off the mutex when 49.1135 + // performing long-duration actions. 49.1136 + 49.1137 + if (mHttpRequestSet.size() < sRequestHighWater 49.1138 + && (! mSkinRequests.empty() 49.1139 + || ! mDecompositionRequests.empty() 49.1140 + || ! mPhysicsShapeRequests.empty())) 49.1141 + { 49.1142 + // Something to do probably, lock and double-check. We don't want 49.1143 + // to hold the lock long here. That will stall main thread activities 49.1144 + // so we bounce it. 49.1145 + 49.1146 + mMutex->lock(); 49.1147 + if (! mSkinRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) 49.1148 + { 49.1149 std::set<LLUUID> incomplete; 49.1150 - for (std::set<LLUUID>::iterator iter = mSkinRequests.begin(); iter != mSkinRequests.end(); ++iter) 49.1151 + std::set<LLUUID>::iterator iter(mSkinRequests.begin()); 49.1152 + while (iter != mSkinRequests.end() && mHttpRequestSet.size() < sRequestHighWater) 49.1153 { 49.1154 LLUUID mesh_id = *iter; 49.1155 - if (!fetchMeshSkinInfo(mesh_id)) 49.1156 + mSkinRequests.erase(iter); 49.1157 + mMutex->unlock(); 49.1158 + 49.1159 + if (! fetchMeshSkinInfo(mesh_id)) 49.1160 { 49.1161 incomplete.insert(mesh_id); 49.1162 } 49.1163 + 49.1164 + mMutex->lock(); 49.1165 + iter = mSkinRequests.begin(); 49.1166 } 49.1167 - mSkinRequests = incomplete; 49.1168 + 49.1169 + if (! incomplete.empty()) 49.1170 + { 49.1171 + mSkinRequests.insert(incomplete.begin(), incomplete.end()); 49.1172 + } 49.1173 } 49.1174 49.1175 - { //mDecompositionRequests is protected by mSignal 49.1176 + // holding lock, try next list 49.1177 + // *TODO: For UI/debug-oriented lists, we might drop the fine- 49.1178 + // grained locking as there's a lowered expectation of smoothness 49.1179 + // in these cases. 49.1180 + if (! mDecompositionRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) 49.1181 + { 49.1182 std::set<LLUUID> incomplete; 49.1183 - for (std::set<LLUUID>::iterator iter = mDecompositionRequests.begin(); iter != mDecompositionRequests.end(); ++iter) 49.1184 + std::set<LLUUID>::iterator iter(mDecompositionRequests.begin()); 49.1185 + while (iter != mDecompositionRequests.end() && mHttpRequestSet.size() < sRequestHighWater) 49.1186 { 49.1187 LLUUID mesh_id = *iter; 49.1188 - if (!fetchMeshDecomposition(mesh_id)) 49.1189 + mDecompositionRequests.erase(iter); 49.1190 + mMutex->unlock(); 49.1191 + 49.1192 + if (! fetchMeshDecomposition(mesh_id)) 49.1193 { 49.1194 incomplete.insert(mesh_id); 49.1195 } 49.1196 + 49.1197 + mMutex->lock(); 49.1198 + iter = mDecompositionRequests.begin(); 49.1199 } 49.1200 - mDecompositionRequests = incomplete; 49.1201 + 49.1202 + if (! incomplete.empty()) 49.1203 + { 49.1204 + mDecompositionRequests.insert(incomplete.begin(), incomplete.end()); 49.1205 + } 49.1206 } 49.1207 49.1208 - { //mPhysicsShapeRequests is protected by mSignal 49.1209 + // holding lock, final list 49.1210 + if (! mPhysicsShapeRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) 49.1211 + { 49.1212 std::set<LLUUID> incomplete; 49.1213 - for (std::set<LLUUID>::iterator iter = mPhysicsShapeRequests.begin(); iter != mPhysicsShapeRequests.end(); ++iter) 49.1214 + std::set<LLUUID>::iterator iter(mPhysicsShapeRequests.begin()); 49.1215 + while (iter != mPhysicsShapeRequests.end() && mHttpRequestSet.size() < sRequestHighWater) 49.1216 { 49.1217 LLUUID mesh_id = *iter; 49.1218 - if (!fetchMeshPhysicsShape(mesh_id)) 49.1219 + mPhysicsShapeRequests.erase(iter); 49.1220 + mMutex->unlock(); 49.1221 + 49.1222 + if (! fetchMeshPhysicsShape(mesh_id)) 49.1223 { 49.1224 incomplete.insert(mesh_id); 49.1225 } 49.1226 + 49.1227 + mMutex->lock(); 49.1228 + iter = mPhysicsShapeRequests.begin(); 49.1229 } 49.1230 - mPhysicsShapeRequests = incomplete; 49.1231 + 49.1232 + if (! incomplete.empty()) 49.1233 + { 49.1234 + mPhysicsShapeRequests.insert(incomplete.begin(), incomplete.end()); 49.1235 + } 49.1236 } 49.1237 - 49.1238 - mCurlRequest->process(); 49.1239 + mMutex->unlock(); 49.1240 } 49.1241 + 49.1242 + // For dev purposes only. A dynamic change could make this false 49.1243 + // and that shouldn't assert. 49.1244 + // llassert_always(mHttpRequestSet.size() <= sRequestHighWater); 49.1245 } 49.1246 49.1247 if (mSignal->isLocked()) 49.1248 @@ -685,25 +1007,25 @@ 49.1249 res = LLConvexDecomposition::quitThread(); 49.1250 if (res != LLCD_OK) 49.1251 { 49.1252 - llwarns << "convex decomposition unable to be quit" << llendl; 49.1253 + LL_WARNS(LOG_MESH) << "Convex decomposition unable to be quit." << LL_ENDL; 49.1254 } 49.1255 - 49.1256 - delete mCurlRequest; 49.1257 - mCurlRequest = NULL; 49.1258 } 49.1259 49.1260 +// Mutex: LLMeshRepoThread::mMutex must be held on entry 49.1261 void LLMeshRepoThread::loadMeshSkinInfo(const LLUUID& mesh_id) 49.1262 -{ //protected by mSignal, no locking needed here 49.1263 +{ 49.1264 mSkinRequests.insert(mesh_id); 49.1265 } 49.1266 49.1267 +// Mutex: LLMeshRepoThread::mMutex must be held on entry 49.1268 void LLMeshRepoThread::loadMeshDecomposition(const LLUUID& mesh_id) 49.1269 -{ //protected by mSignal, no locking needed here 49.1270 +{ 49.1271 mDecompositionRequests.insert(mesh_id); 49.1272 } 49.1273 49.1274 +// Mutex: LLMeshRepoThread::mMutex must be held on entry 49.1275 void LLMeshRepoThread::loadMeshPhysicsShape(const LLUUID& mesh_id) 49.1276 -{ //protected by mSignal, no locking needed here 49.1277 +{ 49.1278 mPhysicsShapeRequests.insert(mesh_id); 49.1279 } 49.1280 49.1281 @@ -716,7 +1038,6 @@ 49.1282 } 49.1283 49.1284 49.1285 - 49.1286 void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod) 49.1287 { //could be called from any thread 49.1288 LLMutexLock lock(mMutex); 49.1289 @@ -748,31 +1069,122 @@ 49.1290 } 49.1291 } 49.1292 49.1293 -//static 49.1294 -std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) 49.1295 +// Mutex: must be holding mMutex when called 49.1296 +void LLMeshRepoThread::setGetMeshCaps(const std::string & get_mesh1, 49.1297 + const std::string & get_mesh2, 49.1298 + int pref_version) 49.1299 { 49.1300 - std::string http_url; 49.1301 + mGetMeshCapability = get_mesh1; 49.1302 + mGetMesh2Capability = get_mesh2; 49.1303 + mGetMeshVersion = pref_version; 49.1304 +} 49.1305 + 49.1306 + 49.1307 +// Constructs a Cap URL for the mesh. Prefers a GetMesh2 cap 49.1308 +// over a GetMesh cap. 49.1309 +// 49.1310 +// Mutex: acquires mMutex 49.1311 +void LLMeshRepoThread::constructUrl(LLUUID mesh_id, std::string * url, int * version) 49.1312 +{ 49.1313 + std::string res_url; 49.1314 + int res_version(2); 49.1315 49.1316 if (gAgent.getRegion()) 49.1317 { 49.1318 - http_url = gMeshRepo.mGetMeshCapability; 49.1319 + LLMutexLock lock(mMutex); 49.1320 + 49.1321 + // Get a consistent pair of (cap string, version). The 49.1322 + // locking could be eliminated here without loss of safety 49.1323 + // by using a set of staging values in setGetMeshCaps(). 49.1324 + 49.1325 + if (! mGetMesh2Capability.empty() && mGetMeshVersion > 1) 49.1326 + { 49.1327 + res_url = mGetMesh2Capability; 49.1328 + res_version = 2; 49.1329 + } 49.1330 + else 49.1331 + { 49.1332 + res_url = mGetMeshCapability; 49.1333 + res_version = 1; 49.1334 + } 49.1335 } 49.1336 49.1337 - if (!http_url.empty()) 49.1338 + if (! res_url.empty()) 49.1339 { 49.1340 - http_url += "/?mesh_id="; 49.1341 - http_url += mesh_id.asString().c_str(); 49.1342 + res_url += "/?mesh_id="; 49.1343 + res_url += mesh_id.asString().c_str(); 49.1344 } 49.1345 else 49.1346 { 49.1347 - llwarns << "Current region does not have GetMesh capability! Cannot load " << mesh_id << ".mesh" << llendl; 49.1348 + LL_WARNS_ONCE(LOG_MESH) << "Current region does not have GetMesh capability! Cannot load " 49.1349 + << mesh_id << ".mesh" << LL_ENDL; 49.1350 } 49.1351 49.1352 - return http_url; 49.1353 + *url = res_url; 49.1354 + *version = res_version; 49.1355 } 49.1356 49.1357 +// Issue an HTTP GET request with byte range using the right 49.1358 +// policy class. Large requests go to the large request class. 49.1359 +// If the current region supports GetMesh2, we prefer that for 49.1360 +// smaller requests otherwise we try to use the traditional 49.1361 +// GetMesh capability and connection concurrency. 49.1362 +// 49.1363 +// @return Valid handle or LLCORE_HTTP_HANDLE_INVALID. 49.1364 +// If the latter, actual status is found in 49.1365 +// mHttpStatus member which is valid until the 49.1366 +// next call to this method. 49.1367 +// 49.1368 +// Thread: repo 49.1369 +LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int cap_version, 49.1370 + size_t offset, size_t len, 49.1371 + LLCore::HttpHandler * handler) 49.1372 +{ 49.1373 + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); 49.1374 + 49.1375 + if (len < LARGE_MESH_FETCH_THRESHOLD) 49.1376 + { 49.1377 + handle = mHttpRequest->requestGetByteRange((2 == cap_version 49.1378 + ? mHttpPolicyClass 49.1379 + : mHttpLegacyPolicyClass), 49.1380 + mHttpPriority, 49.1381 + url, 49.1382 + offset, 49.1383 + len, 49.1384 + mHttpOptions, 49.1385 + mHttpHeaders, 49.1386 + handler); 49.1387 + if (LLCORE_HTTP_HANDLE_INVALID != handle) 49.1388 + { 49.1389 + ++LLMeshRepository::sHTTPRequestCount; 49.1390 + } 49.1391 + } 49.1392 + else 49.1393 + { 49.1394 + handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, 49.1395 + mHttpPriority, 49.1396 + url, 49.1397 + offset, 49.1398 + len, 49.1399 + mHttpLargeOptions, 49.1400 + mHttpHeaders, 49.1401 + handler); 49.1402 + if (LLCORE_HTTP_HANDLE_INVALID != handle) 49.1403 + { 49.1404 + ++LLMeshRepository::sHTTPLargeRequestCount; 49.1405 + } 49.1406 + } 49.1407 + if (LLCORE_HTTP_HANDLE_INVALID == handle) 49.1408 + { 49.1409 + // Something went wrong, capture the error code for caller. 49.1410 + mHttpStatus = mHttpRequest->getStatus(); 49.1411 + } 49.1412 + return handle; 49.1413 +} 49.1414 + 49.1415 + 49.1416 bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) 49.1417 -{ //protected by mMutex 49.1418 +{ 49.1419 49.1420 if (!mHeaderMutex) 49.1421 { 49.1422 @@ -787,7 +1199,8 @@ 49.1423 return false; 49.1424 } 49.1425 49.1426 - bool ret = true ; 49.1427 + ++LLMeshRepository::sMeshRequestCount; 49.1428 + bool ret = true; 49.1429 U32 header_size = mMeshHeaderSize[mesh_id]; 49.1430 49.1431 if (header_size > 0) 49.1432 @@ -805,6 +1218,7 @@ 49.1433 if (file.getSize() >= offset+size) 49.1434 { 49.1435 LLMeshRepository::sCacheBytesRead += size; 49.1436 + ++LLMeshRepository::sCacheReads; 49.1437 file.seek(offset); 49.1438 U8* buffer = new U8[size]; 49.1439 file.read(buffer, size); 49.1440 @@ -829,17 +1243,28 @@ 49.1441 } 49.1442 49.1443 //reading from VFS failed for whatever reason, fetch from sim 49.1444 - std::vector<std::string> headers; 49.1445 - headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); 49.1446 - 49.1447 - std::string http_url = constructUrl(mesh_id); 49.1448 + int cap_version(2); 49.1449 + std::string http_url; 49.1450 + constructUrl(mesh_id, &http_url, &cap_version); 49.1451 + 49.1452 if (!http_url.empty()) 49.1453 - { 49.1454 - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, 49.1455 - new LLMeshSkinInfoResponder(mesh_id, offset, size)); 49.1456 - if(ret) 49.1457 + { 49.1458 + LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size); 49.1459 + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); 49.1460 + if (LLCORE_HTTP_HANDLE_INVALID == handle) 49.1461 { 49.1462 - LLMeshRepository::sHTTPRequestCount++; 49.1463 + LL_WARNS(LOG_MESH) << "HTTP GET request failed for skin info on mesh " << mID 49.1464 + << ". Reason: " << mHttpStatus.toString() 49.1465 + << " (" << mHttpStatus.toTerseString() << ")" 49.1466 + << LL_ENDL; 49.1467 + delete handler; 49.1468 + ret = false; 49.1469 + 49.1470 + } 49.1471 + else 49.1472 + { 49.1473 + handler->mHttpHandle = handle; 49.1474 + mHttpRequestSet.insert(handler); 49.1475 } 49.1476 } 49.1477 } 49.1478 @@ -854,7 +1279,7 @@ 49.1479 } 49.1480 49.1481 bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) 49.1482 -{ //protected by mMutex 49.1483 +{ 49.1484 if (!mHeaderMutex) 49.1485 { 49.1486 return false; 49.1487 @@ -868,8 +1293,9 @@ 49.1488 return false; 49.1489 } 49.1490 49.1491 + ++LLMeshRepository::sMeshRequestCount; 49.1492 U32 header_size = mMeshHeaderSize[mesh_id]; 49.1493 - bool ret = true ; 49.1494 + bool ret = true; 49.1495 49.1496 if (header_size > 0) 49.1497 { 49.1498 @@ -886,6 +1312,7 @@ 49.1499 if (file.getSize() >= offset+size) 49.1500 { 49.1501 LLMeshRepository::sCacheBytesRead += size; 49.1502 + ++LLMeshRepository::sCacheReads; 49.1503 49.1504 file.seek(offset); 49.1505 U8* buffer = new U8[size]; 49.1506 @@ -911,17 +1338,27 @@ 49.1507 } 49.1508 49.1509 //reading from VFS failed for whatever reason, fetch from sim 49.1510 - std::vector<std::string> headers; 49.1511 - headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); 49.1512 - 49.1513 - std::string http_url = constructUrl(mesh_id); 49.1514 + int cap_version(2); 49.1515 + std::string http_url; 49.1516 + constructUrl(mesh_id, &http_url, &cap_version); 49.1517 + 49.1518 if (!http_url.empty()) 49.1519 - { 49.1520 - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, 49.1521 - new LLMeshDecompositionResponder(mesh_id, offset, size)); 49.1522 - if(ret) 49.1523 + { 49.1524 + LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size); 49.1525 + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); 49.1526 + if (LLCORE_HTTP_HANDLE_INVALID == handle) 49.1527 { 49.1528 - LLMeshRepository::sHTTPRequestCount++; 49.1529 + LL_WARNS(LOG_MESH) << "HTTP GET request failed for decomposition mesh " << mID 49.1530 + << ". Reason: " << mHttpStatus.toString() 49.1531 + << " (" << mHttpStatus.toTerseString() << ")" 49.1532 + << LL_ENDL; 49.1533 + delete handler; 49.1534 + ret = false; 49.1535 + } 49.1536 + else 49.1537 + { 49.1538 + handler->mHttpHandle = handle; 49.1539 + mHttpRequestSet.insert(handler); 49.1540 } 49.1541 } 49.1542 } 49.1543 @@ -936,7 +1373,7 @@ 49.1544 } 49.1545 49.1546 bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) 49.1547 -{ //protected by mMutex 49.1548 +{ 49.1549 if (!mHeaderMutex) 49.1550 { 49.1551 return false; 49.1552 @@ -950,8 +1387,9 @@ 49.1553 return false; 49.1554 } 49.1555 49.1556 + ++LLMeshRepository::sMeshRequestCount; 49.1557 U32 header_size = mMeshHeaderSize[mesh_id]; 49.1558 - bool ret = true ; 49.1559 + bool ret = true; 49.1560 49.1561 if (header_size > 0) 49.1562 { 49.1563 @@ -968,6 +1406,7 @@ 49.1564 if (file.getSize() >= offset+size) 49.1565 { 49.1566 LLMeshRepository::sCacheBytesRead += size; 49.1567 + ++LLMeshRepository::sCacheReads; 49.1568 file.seek(offset); 49.1569 U8* buffer = new U8[size]; 49.1570 file.read(buffer, size); 49.1571 @@ -992,18 +1431,27 @@ 49.1572 } 49.1573 49.1574 //reading from VFS failed for whatever reason, fetch from sim 49.1575 - std::vector<std::string> headers; 49.1576 - headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); 49.1577 - 49.1578 - std::string http_url = constructUrl(mesh_id); 49.1579 + int cap_version(2); 49.1580 + std::string http_url; 49.1581 + constructUrl(mesh_id, &http_url, &cap_version); 49.1582 + 49.1583 if (!http_url.empty()) 49.1584 - { 49.1585 - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, 49.1586 - new LLMeshPhysicsShapeResponder(mesh_id, offset, size)); 49.1587 - 49.1588 - if(ret) 49.1589 + { 49.1590 + LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size); 49.1591 + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); 49.1592 + if (LLCORE_HTTP_HANDLE_INVALID == handle) 49.1593 { 49.1594 - LLMeshRepository::sHTTPRequestCount++; 49.1595 + LL_WARNS(LOG_MESH) << "HTTP GET request failed for physics shape on mesh " << mID 49.1596 + << ". Reason: " << mHttpStatus.toString() 49.1597 + << " (" << mHttpStatus.toTerseString() << ")" 49.1598 + << LL_ENDL; 49.1599 + delete handler; 49.1600 + ret = false; 49.1601 + } 49.1602 + else 49.1603 + { 49.1604 + handler->mHttpHandle = handle; 49.1605 + mHttpRequestSet.insert(handler); 49.1606 } 49.1607 } 49.1608 } 49.1609 @@ -1050,8 +1498,10 @@ 49.1610 } 49.1611 49.1612 //return false if failed to get header 49.1613 -bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count) 49.1614 +bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params) 49.1615 { 49.1616 + ++LLMeshRepository::sMeshRequestCount; 49.1617 + 49.1618 { 49.1619 //look for mesh in asset in vfs 49.1620 LLVFile file(gVFS, mesh_params.getSculptID(), LLAssetType::AT_MESH); 49.1621 @@ -1059,43 +1509,57 @@ 49.1622 S32 size = file.getSize(); 49.1623 49.1624 if (size > 0) 49.1625 - { //NOTE -- if the header size is ever more than 4KB, this will break 49.1626 - U8 buffer[4096]; 49.1627 - S32 bytes = llmin(size, 4096); 49.1628 + { 49.1629 + // *NOTE: if the header size is ever more than 4KB, this will break 49.1630 + U8 buffer[MESH_HEADER_SIZE]; 49.1631 + S32 bytes = llmin(size, MESH_HEADER_SIZE); 49.1632 LLMeshRepository::sCacheBytesRead += bytes; 49.1633 + ++LLMeshRepository::sCacheReads; 49.1634 file.read(buffer, bytes); 49.1635 if (headerReceived(mesh_params, buffer, bytes)) 49.1636 - { //did not do an HTTP request, return false 49.1637 + { 49.1638 + // Found mesh in VFS cache 49.1639 return true; 49.1640 } 49.1641 } 49.1642 } 49.1643 49.1644 //either cache entry doesn't exist or is corrupt, request header from simulator 49.1645 - bool retval = true ; 49.1646 - std::vector<std::string> headers; 49.1647 - headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); 49.1648 - 49.1649 - std::string http_url = constructUrl(mesh_params.getSculptID()); 49.1650 + bool retval = true; 49.1651 + int cap_version(2); 49.1652 + std::string http_url; 49.1653 + constructUrl(mesh_params.getSculptID(), &http_url, &cap_version); 49.1654 + 49.1655 if (!http_url.empty()) 49.1656 { 49.1657 //grab first 4KB if we're going to bother with a fetch. Cache will prevent future fetches if a full mesh fits 49.1658 //within the first 4KB 49.1659 //NOTE -- this will break of headers ever exceed 4KB 49.1660 - retval = mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params)); 49.1661 - if(retval) 49.1662 + 49.1663 + LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); 49.1664 + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler); 49.1665 + if (LLCORE_HTTP_HANDLE_INVALID == handle) 49.1666 { 49.1667 - LLMeshRepository::sHTTPRequestCount++; 49.1668 + LL_WARNS(LOG_MESH) << "HTTP GET request failed for mesh header " << mID 49.1669 + << ". Reason: " << mHttpStatus.toString() 49.1670 + << " (" << mHttpStatus.toTerseString() << ")" 49.1671 + << LL_ENDL; 49.1672 + delete handler; 49.1673 + retval = false; 49.1674 } 49.1675 - count++; 49.1676 + else 49.1677 + { 49.1678 + handler->mHttpHandle = handle; 49.1679 + mHttpRequestSet.insert(handler); 49.1680 + } 49.1681 } 49.1682 49.1683 return retval; 49.1684 } 49.1685 49.1686 //return false if failed to get mesh lod. 49.1687 -bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count) 49.1688 -{ //protected by mMutex 49.1689 +bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod) 49.1690 +{ 49.1691 if (!mHeaderMutex) 49.1692 { 49.1693 return false; 49.1694 @@ -1103,6 +1567,7 @@ 49.1695 49.1696 mHeaderMutex->lock(); 49.1697