// // well, imported. But this gets StyleCop off our back
//-----------------------------------------------------------------------
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
// Vance Morrison
//-----------------------------------------------------------------------
namespace DotNetOpenAuth.Test.Performance {
using System;
using System.Collections.Generic;
using System.Diagnostics;
///
/// Stats represents a list of samples (floating point values) This class can calculate the standard
/// statistics on this list (Mean, Median, StandardDeviation ...)
///
internal class Stats : IEnumerable {
private List data;
private float minimum;
private float maximum;
private float median;
private float mean;
private float standardDeviation;
private bool statsComputed;
public Stats() { data = new List(); }
public void Add(float dataItem) {
statsComputed = false;
data.Add(dataItem);
}
public void RemoveRange(int index, int count) {
data.RemoveRange(index, count);
statsComputed = false;
}
internal void Adjust(float delta) {
statsComputed = false;
for (int i = 0; i < data.Count; i++)
data[i] += delta;
}
internal void AdjustForScale(float scale) {
statsComputed = false;
for (int i = 0; i < data.Count; i++)
data[i] /= scale;
}
public int Count { get { return data.Count; } }
public float this[int idx] { get { return data[idx]; } }
public IEnumerator GetEnumerator() { return data.GetEnumerator(); }
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return data.GetEnumerator(); }
public float Minimum {
get {
if (!statsComputed) {
this.ComputeStats();
}
return minimum;
}
}
public float Maximum {
get {
if (!statsComputed) {
this.ComputeStats();
}
return maximum;
}
}
public float Median {
get {
if (!statsComputed) {
this.ComputeStats();
}
return median;
}
}
public float Mean {
get {
if (!statsComputed) {
this.ComputeStats();
}
return mean;
}
}
public float StandardDeviation {
get {
if (!statsComputed) {
this.ComputeStats();
}
return standardDeviation;
}
}
///
/// Returns a that represents this instance.
///
///
/// A that represents this instance.
///
public override string ToString() {
if (!statsComputed) {
ComputeStats();
}
return "mean=" + mean.ToString("f3") + " median=" + median.ToString("f3") +
" min=" + minimum.ToString("f3") + " max=" + maximum.ToString("f3") +
" sdtdev=" + standardDeviation.ToString("f3") + " samples=" + Count.ToString();
}
private void ComputeStats() {
minimum = float.MaxValue;
maximum = float.MinValue;
median = 0.0F;
mean = 0.0F;
standardDeviation = 0.0F;
double total = 0;
foreach (float dataPoint in this) {
if (dataPoint < minimum)
minimum = dataPoint;
if (dataPoint > maximum)
maximum = dataPoint;
total += dataPoint;
}
if (Count > 0) {
data.Sort();
if (Count % 2 == 1)
median = this[Count / 2];
else
median = (this[(Count / 2) - 1] + this[Count / 2]) / 2;
mean = (float)(total / Count);
double squares = 0.0;
foreach (float dataPoint in this) {
double diffFromMean = dataPoint - mean;
squares += diffFromMean * diffFromMean;
}
standardDeviation = (float)Math.Sqrt(squares / Count);
}
statsComputed = true;
}
};
///
/// The CodeTimer class only times one invocation of the code. Often, you want to collect many samples so
/// that you can determine how noisy the resulting data is. This is what MultiSampleCodeTimer does.
///
internal class MultiSampleCodeTimer {
public MultiSampleCodeTimer() : this(1) { }
public MultiSampleCodeTimer(int sampleCount) : this(sampleCount, 1) { }
public MultiSampleCodeTimer(int sampleCount, int iterationCount) {
SampleCount = sampleCount;
timer = new CodeTimer(iterationCount);
timer.Prime = false; // We will do the priming (or not).
Prime = true;
}
public MultiSampleCodeTimer(MultiSampleCodeTimer template)
: this(template.SampleCount, template.IterationCount) {
OnMeasure = template.OnMeasure;
}
///
/// If true (the default), the benchmark is run once before the actual measurement to
/// insure that any 'first time' initialization is complete.
///
public bool Prime;
///
/// The number of times the benchmark is run in a loop for a single measument.
///
public int IterationCount { get { return timer.IterationCount; } set { timer.IterationCount = value; } }
///
/// The number of measurments to make for a single benchmark.
///
public int SampleCount;
///
/// The smallest time (in microseconds) that can be resolved by the timer).
///
public static float ResolutionUsec { get { return 1000000.0F / Stopwatch.Frequency; } }
public delegate void MeasureCallback(string name, int iterationCount, float scale, Stats sample);
///
/// OnMeasure is signaled every time a Measure() is called.
///
public event MeasureCallback OnMeasure;
public Stats Measure(string name, Action action) {
return Measure(name, 1, action, null);
}
///
/// The main measurment routine. Calling this will cause code:OnMeasure event to be
/// raised.
///
/// name of the benchmark
/// The number of times the benchmark is cloned in 'action' (typically 1)
/// The actual code to measure.
/// A Stats object representing the measurements (in usec)
public Stats Measure(string name, float scale, Action action) {
return Measure(name, scale, action, null);
}
///
/// The main measurment routine. Calling this will cause code:OnMeasure event to be
/// raised.
///
/// name of the benchmark
/// The number of times the benchmark is cloned in 'action' (typically 1)
/// The actual code to measure.
/// Code that will be called before 'action' to reset the state of the benchmark.
/// A Stats object representing the measurements (in usec)
public Stats Measure(string name, float scale, Action action, Action reset) {
if (reset != null && IterationCount != 1)
throw new ApplicationException("Reset can only be used on timers with an iteration count of 1");
Stats statsUSec = new Stats();
if (Prime) {
if (reset != null)
reset();
action();
}
for (int i = 0; i < SampleCount; i++) {
if (reset != null)
reset();
statsUSec.Add(timer.Measure(name, scale, action));
}
if (OnMeasure != null)
OnMeasure(name, IterationCount, scale, statsUSec);
return statsUSec;
}
///
/// Prints the mean, median, min, max, and stdDev and count of the samples to the Console
/// Useful as a target for OnMeasure
///
public static MeasureCallback PrintStats = delegate(string name, int iterationCount, float scale, Stats sample) {
Console.WriteLine(name + ": " + sample.ToString());
};
///
/// Prints the mean with a error bound (2 standard deviations, which imply a you have
/// 95% confidence that a sampleUsec will be with the bounds (for a normal distribution).
/// This is a good default target for OnMeasure.
///
public static MeasureCallback Print = delegate(string name, int iterationCount, float scale, Stats sample) {
// +- two standard deviations covers 95% of all samples in a normal distribution
float errorPercent = (sample.StandardDeviation * 2 * 100) / Math.Abs(sample.Mean);
string errorString = ">400%";
if (errorPercent < 400)
errorString = (errorPercent.ToString("f0") + "%").PadRight(5);
string countString = "";
if (iterationCount != 1)
countString = "count: " + iterationCount.ToString() + " ";
Console.WriteLine(name + ": " + countString + sample.Mean.ToString("f3").PadLeft(8) + " +- " + errorString + " msec");
};
#region privates
CodeTimer timer;
#endregion
};
///
/// CodeTimer is a simple wrapper that uses System.Diagnostics.StopWatch
/// to time the body of some code (given by a delegate), to high precision.
///
public class CodeTimer {
public CodeTimer() : this(1) { }
public CodeTimer(int iterationCount) {
this.iterationCount = iterationCount;
Prime = true;
// Spin the CPU for a while. This should help insure that the CPU gets out of any low power
// mode so so that we get more stable results.
// TODO: see if this is true, and if there is a better way of doing it.
Stopwatch sw = Stopwatch.StartNew();
while (sw.ElapsedMilliseconds < 32) {
}
}
///
/// The number of times the benchmark is run in a loop for a single measument.
///
public int IterationCount {
get { return iterationCount; }
set {
iterationCount = value;
overheadValid = false;
}
}
///
/// By default CodeTimer will run the action once before doing a
/// measurement run. This insures one-time actions like JIT
/// compilation are not being measured. However if the benchmark is
/// not idempotent, this can be a problem. Setting Prime=false
/// insures that this Priming does not happen.
///
public bool Prime;
public delegate void MeasureCallback(string name, int iterationCount, float sample);
///
/// OnMeasure is signaled every time a Measure() is called.
///
public event MeasureCallback OnMeasure;
///
/// The smallest time (in microseconds) that can be resolved by the timer).
///
public static float ResolutionUsec { get { return 1000000.0F / Stopwatch.Frequency; } }
///
/// Returns the number of microsecond it took to run 'action', 'count' times.
///
public float Measure(string name, Action action) {
return Measure(name, 1, action);
}
///
/// Returns the number of microseconds it to to run action 'count' times divided by 'scale'.
/// Scaling is useful if you want to normalize to a single iteration for example.
///
public float Measure(string name, float scale, Action action) {
Stopwatch sw = new Stopwatch();
// Run the action once to do any JITTing that might happen.
if (Prime)
action();
float overheadUsec = GetOverheadUsec(action);
sw.Reset();
sw.Start();
for (int j = 0; j < iterationCount; j++)
action();
sw.Stop();
float sampleUsec = (float)((sw.Elapsed.TotalMilliseconds * 1000.0F - overheadUsec) / scale / iterationCount);
if (!computingOverhead && OnMeasure != null)
OnMeasure(name, iterationCount, sampleUsec);
return sampleUsec;
}
///
/// Prints the result of a CodeTimer to standard output.
/// This is a good default target for OnMeasure.
///
public static MeasureCallback Print = delegate(string name, int iterationCount, float sample) {
Console.WriteLine("{0}: count={1} time={2:f3} msec ", name, iterationCount, sample);
};
#region privates
///
/// Time the overheadUsec of the harness that does nothing so we can subtract it out.
///
/// Because calling delegates on static methods is more expensive than caling delegates on
/// instance methods we need the action to determine the overheadUsec.
///
///
float GetOverheadUsec(Action action) {
if (!overheadValid) {
if (computingOverhead)
return 0.0F;
computingOverhead = true;
// Compute the overheads of calling differnet types of delegates.
Action emptyInstanceAction = new Action(this.emptyMethod);
// Prime the actions (JIT them)
Measure(null, emptyInstanceAction);
// Compute the min over 5 runs (figuring better not to go negative)
instanceOverheadUsec = float.MaxValue;
for (int i = 0; i < 5; i++) {
// We multiply by iteration count because we don't want this scaled by the
// count but 'Measure' does it by whether we want it or not.
instanceOverheadUsec = Math.Min(Measure(null, emptyInstanceAction) * IterationCount, instanceOverheadUsec);
}
Action emptyStaticAction = new Action(emptyStaticMethod);
Measure(null, emptyStaticAction);
staticOverheadUsec = float.MaxValue;
for (int i = 0; i < 5; i++)
staticOverheadUsec = Math.Min(Measure(null, emptyStaticAction) * IterationCount, staticOverheadUsec);
computingOverhead = false;
overheadValid = true;
}
if (action.Target == null)
return staticOverheadUsec;
else
return instanceOverheadUsec;
}
static private void emptyStaticMethod() { }
private void emptyMethod() { }
bool overheadValid;
bool computingOverhead;
int iterationCount;
float staticOverheadUsec;
float instanceOverheadUsec;
#endregion
};
}