Managing goal-based load using multiple perfcounter thresholds
As promised, here is a sample plug-in that I wrote a few months ago which people here may find useful. The purpose of this plug-in is to expand the functionality of goal-based load which ships with Visual Studio team test to allow you to have your load step up or down based on any number of performance counters.
For the plug-in to work, you need to have a few things set on your loadtest.
Thresholds:
For each perfcounter you intend to track, you need to set the warning and critical threshold values on that perfcounter.
Context variables:
MinLoad: <The Minimum amount of load you wish to apply, regardless of perfcounter thresholds.>
MaxLoad: <The maximum amount of load you wish to apply, regardless of perfcounter thresholds.>
LoadStep: <The amount of load to step when increasing or decreasing.>
LoadChangeInterval: <How often you wish to change the load (in seconds and if necessary)>
PerfCounter<#>: A path to the performance counter in the form: <machine>\<category>\<instance>\<counter> For example: COMPUTER_NAME\Processor\_Total\% Processor Time
Note that you can have as many perfcounters as you wish to track, however you need to make sure they are sequential in number. (in other words, if you have PerfCounter1, PerfCounter2, and PerfCounter4 - PerfCounter4 will not be tracked.)
Plug-in Code:
using System;
using System.Collections;
using Microsoft.VisualStudio.TestTools.LoadTesting;
/*
* VariableLoadPlugin - Useful for using multiple performance counters with goal
* based load profiles.
*
* Usage: Assign the following context parameters to your loadtest
* MinLoad - The minimum amount of load to generate.
* MaxLoad - The maximum amount of load to generate.
* LoadStep - The amount of load increment / decrement by when attempting
* to reach your target thresholds.
* LoadChangeInterval - The number of seconds between load changes
* Perfcounter# - A performance counter you wish tracked. The value should
* be of the form: <computer>\<category>\<instance>\<counter>
* For example: Perfcounter1 : \MYCOMPUTER\Processor\_Total\% Processor Time
*
* Notes:
*
* 1) You can have as many perfcounters as you want, they just need to have sequential
* context values. (e.g. Perfcounter1, Perfcounter2, Perfcounter3, etc)
* 2) The plugin relies on Warning and Critical ThresholdExceeded events to fire in order
* to adjust the load. That means you need to set Warning and Critical threshold values
* as part of your loadtest for the perfcounters you want tracked.
* 3) There is a bug in the RTM version of VSTT in which ThresholdExceeded events do
* not fire reliably when a unit test is part of the loadtest mix. For this reason, it's
* recommented that you don't use unit tests with this plugin.
*/
namespace LoadtestPlugins
{
public class VariableLoadPlugin : ILoadTestPlugin
{
private ThresholdTracker m_thresholdTracker;
private int m_minLoad;
private int m_maxLoad;
private int m_loadStep;
private LoadTest m_loadTest;
/*
* The loadtest plugin initialize event - Used to parse the user's
* data out of the context object and set up the ThresholdTracker object
* as necessary.
*/
public void Initialize(LoadTest loadTest)
{
int contextIndex = 0;
int intervalLength = 0;
string currentCounter;
m_loadTest = loadTest;
/*
* Assign MinLoad / MaxLoad / LoadStep and Interval based on context variables.
* If they don't exist, assign defaults of 1, 1000, 10, and 20 respectively.
*/
AssignContextValueIfAvailable("MinLoad", ref m_minLoad, 1);
AssignContextValueIfAvailable("MaxLoad", ref m_maxLoad, 1000);
AssignContextValueIfAvailable("LoadStep", ref m_loadStep, 10);
AssignContextValueIfAvailable("LoadChangeInterval", ref intervalLength, 20);
if (null == m_thresholdTracker)
{
m_thresholdTracker = new ThresholdTracker();
currentCounter = String.Format("PerfCounter{0}", ++contextIndex);
while (m_loadTest.Context.ContainsKey(currentCounter))
{
// Add a lowercase version of the perfcounter into the ThresholdTracker
m_thresholdTracker.AddTrackedPerformanceCounter(m_loadTest.Context[currentCounter].ToString().ToLower());
currentCounter = String.Format("PerfCounter{0}", ++contextIndex);
}
// Allow the thresholdTracker to start tracking perfcounter data.
m_thresholdTracker.StartTracking(intervalLength);
// Attach the event handlers as necessary.
m_loadTest.ThresholdExceeded += new EventHandler<ThresholdExceededEventArgs>(LoadTest_ThresholdExceeded);
m_loadTest.Heartbeat += new EventHandler<HeartbeatEventArgs>(LoadTest_Heartbeat);
}
}
/*
* The ThresholdExceeded event, executed by the loadtest engine when a perfcounter's
* threshold (warning or critical) is violated.
* This method passes that data into the ThresholdTracker object for consideration.
*/
void LoadTest_ThresholdExceeded(object sender, ThresholdExceededEventArgs e)
{
string counterName;
// Counters with no instance have a placeholder for the instance name
if ("systemdiagnosticsperfcounterlibsingleinstance" == e.CounterValue.InstanceName)
{
counterName = String.Format("{0}\\{1}\\{2}", e.CounterValue.MachineName,
e.CounterValue.CategoryName,
e.CounterValue.CounterName);
}
else
{
counterName = String.Format("{0}\\{1}\\{2}\\{3}", e.CounterValue.MachineName,
e.CounterValue.CategoryName,
e.CounterValue.InstanceName,
e.CounterValue.CounterName);
}
m_thresholdTracker.ReportThresholdViolation(counterName.ToLower(), e.ThresholdResult);
}
/*
* The loadtest heartbeat event, executed once per second.
* This method will check to see if the ThresholdTracker object recommends
* a load change and perform the change.
*/
void LoadTest_Heartbeat(object sender, HeartbeatEventArgs e)
{
int newLoad;
if (m_thresholdTracker.IsStarted)
{
switch (m_thresholdTracker.CurrentLoadDirection)
{
case ThresholdTracker.LoadDirection.None:
break;
case ThresholdTracker.LoadDirection.Decrease:
foreach (LoadTestScenario scenario in m_loadTest.Scenarios)
{
newLoad = scenario.CurrentLoad - m_loadStep;
scenario.CurrentLoad = (newLoad >= m_minLoad) ? newLoad : m_minLoad;
}
break;
case ThresholdTracker.LoadDirection.Increase:
foreach (LoadTestScenario scenario in m_loadTest.Scenarios)
{
newLoad = scenario.CurrentLoad + m_loadStep;
scenario.CurrentLoad = (newLoad <= m_maxLoad) ? newLoad : m_maxLoad;
}
break;
}
}
}
/*
* Helper method for grabbing values from the context data or assigning
* a default value if needed.
* Returns true if the value was found, false if the default value was used.
*/
private bool AssignContextValueIfAvailable(string contextKeyName, ref int target, int defaultValue)
{
if (m_loadTest.Context.ContainsKey(contextKeyName))
{
target = Convert.ToInt32(m_loadTest.Context[contextKeyName]);
return true;
}
target = defaultValue;
return false;
}
}
public class ThresholdTracker
{
public enum LoadDirection
{
Increase = 0,
Decrease = 1,
None = 2,
}
private int m_intervalLength;
private ArrayList m_counterList;
private DateTime m_nextUpdateTime;
private LoadDirection m_loadDirection;
public ThresholdTracker()
{
m_intervalLength = -1;
m_counterList = new ArrayList();
m_loadDirection = LoadDirection.None;
}
/*
* Method for adding new performance counters into the tracking array.
* Note that this does not guarantee that counter thresholds have been set,
* only that that the perfcounter will be tracked if they are.
*/
public bool AddTrackedPerformanceCounter(string perfCounter)
{
if(null != perfCounter && !m_counterList.Contains(perfCounter))
{
m_counterList.Add(perfCounter);
return true;
}
return false;
}
/*
* Once a perfcounter has hit a threshold (be it critical or warning),
* we track it here to determine if load should be affected or not.
*/
public bool ReportThresholdViolation(string counter, ThresholdRuleResult thresholdResult)
{
if (IsStarted && m_counterList.Contains(counter))
{
switch (thresholdResult)
{
case ThresholdRuleResult.OK:
// Do nothing in this case, as we default to increasing load anyway.
break;
case ThresholdRuleResult.Warning:
/*
* Comment this section out if you want to bounce off of the upper end
* of your highest threshold, only decreasing load when you hit your
* critical edge. (This will allow for more perfcounters to sit
* between their warning and critical ranges, but will cause load to
* be changed at every interval)
* Leaving this section uncommented will keep your load at the low end
* of your highest threshold (In other words, you'll stop increasing
* load as soon as your highest tracked perfcounter hits its warning
* threshold)
*/
if (LoadDirection.Increase == m_loadDirection)
{
m_loadDirection = LoadDirection.None;
}
break;
case ThresholdRuleResult.Critical:
// Nothing to do here but decrease the load if we've violated a
// critical threshold.
m_loadDirection = LoadDirection.Decrease;
break;
}
// The perfcounter was being tracked.
return true;
}
// Perfcounter wasn't tracked anyway.
return false;
}
/*
* A property which provides a recommendation on whether to increase
* or decrease the current load on the server in order to hit the
* target perfcounter thresholds
*/
public LoadDirection CurrentLoadDirection
{
get
{
if (!IsStarted)
{
throw new ApplicationException("Threshold tracker has not been started");
}
// No load changes if the interval time hasn't passed.
if (m_nextUpdateTime > DateTime.Now)
{
return LoadDirection.None;
}
// The interval time has passed, let's change the load (if necessary)
m_nextUpdateTime = DateTime.Now.AddSeconds(m_intervalLength);
// Grab a copy of the current value so we can return it.
LoadDirection tempDirection = m_loadDirection;
/*
* Default to increasing load next time. (This will change if a
* threshold is violated.)
*/
m_loadDirection = LoadDirection.Increase;
return tempDirection;
}
}
/*
* Readonly property for quick assessment on whether perfcounters are
* being tracked or not.
*/
public bool IsStarted
{
get
{
return (-1 != m_intervalLength);
}
}
/*
* Allows this object to start tracking perfcounter data at specified
* intervals
*/
public void StartTracking(int intervalLength)
{
//Use the default of 20 seconds if they fail to specify a valid interval.
m_intervalLength = (intervalLength > 0) ? intervalLength : 20;
}
/*
* Forces the class to stop keeping track of perfcounter threshold violations
*/
public void StopTracking()
{
if (!IsStarted)
{
throw new ApplicationException("Threshold tracker has not been started");
}
m_intervalLength = -1;
}
}
}