// // well, imported. But this gets StyleCop off our back //----------------------------------------------------------------------- // // Copyright (c) Microsoft Corporation. All rights reserved. // // Vance Morrison //----------------------------------------------------------------------- namespace DotNetOpenAuth.Test.Performance { using System; using System.Collections.Generic; using System.Diagnostics; /// /// Stats represents a list of samples (floating point values) This class can calculate the standard /// statistics on this list (Mean, Median, StandardDeviation ...) /// internal class Stats : IEnumerable { private List data; private float minimum; private float maximum; private float median; private float mean; private float standardDeviation; private bool statsComputed; public Stats() { data = new List(); } public void Add(float dataItem) { statsComputed = false; data.Add(dataItem); } public void RemoveRange(int index, int count) { data.RemoveRange(index, count); statsComputed = false; } internal void Adjust(float delta) { statsComputed = false; for (int i = 0; i < data.Count; i++) data[i] += delta; } internal void AdjustForScale(float scale) { statsComputed = false; for (int i = 0; i < data.Count; i++) data[i] /= scale; } public int Count { get { return data.Count; } } public float this[int idx] { get { return data[idx]; } } public IEnumerator GetEnumerator() { return data.GetEnumerator(); } System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return data.GetEnumerator(); } public float Minimum { get { if (!statsComputed) { this.ComputeStats(); } return minimum; } } public float Maximum { get { if (!statsComputed) { this.ComputeStats(); } return maximum; } } public float Median { get { if (!statsComputed) { this.ComputeStats(); } return median; } } public float Mean { get { if (!statsComputed) { this.ComputeStats(); } return mean; } } public float StandardDeviation { get { if (!statsComputed) { this.ComputeStats(); } return standardDeviation; } } /// /// Returns a that represents this instance. /// /// /// A that represents this instance. /// public override string ToString() { if (!statsComputed) { ComputeStats(); } return "mean=" + mean.ToString("f3") + " median=" + median.ToString("f3") + " min=" + minimum.ToString("f3") + " max=" + maximum.ToString("f3") + " sdtdev=" + standardDeviation.ToString("f3") + " samples=" + Count.ToString(); } private void ComputeStats() { minimum = float.MaxValue; maximum = float.MinValue; median = 0.0F; mean = 0.0F; standardDeviation = 0.0F; double total = 0; foreach (float dataPoint in this) { if (dataPoint < minimum) minimum = dataPoint; if (dataPoint > maximum) maximum = dataPoint; total += dataPoint; } if (Count > 0) { data.Sort(); if (Count % 2 == 1) median = this[Count / 2]; else median = (this[(Count / 2) - 1] + this[Count / 2]) / 2; mean = (float)(total / Count); double squares = 0.0; foreach (float dataPoint in this) { double diffFromMean = dataPoint - mean; squares += diffFromMean * diffFromMean; } standardDeviation = (float)Math.Sqrt(squares / Count); } statsComputed = true; } }; /// /// The CodeTimer class only times one invocation of the code. Often, you want to collect many samples so /// that you can determine how noisy the resulting data is. This is what MultiSampleCodeTimer does. /// internal class MultiSampleCodeTimer { public MultiSampleCodeTimer() : this(1) { } public MultiSampleCodeTimer(int sampleCount) : this(sampleCount, 1) { } public MultiSampleCodeTimer(int sampleCount, int iterationCount) { SampleCount = sampleCount; timer = new CodeTimer(iterationCount); timer.Prime = false; // We will do the priming (or not). Prime = true; } public MultiSampleCodeTimer(MultiSampleCodeTimer template) : this(template.SampleCount, template.IterationCount) { OnMeasure = template.OnMeasure; } /// /// If true (the default), the benchmark is run once before the actual measurement to /// insure that any 'first time' initialization is complete. /// public bool Prime; /// /// The number of times the benchmark is run in a loop for a single measument. /// public int IterationCount { get { return timer.IterationCount; } set { timer.IterationCount = value; } } /// /// The number of measurments to make for a single benchmark. /// public int SampleCount; /// /// The smallest time (in microseconds) that can be resolved by the timer). /// public static float ResolutionUsec { get { return 1000000.0F / Stopwatch.Frequency; } } public delegate void MeasureCallback(string name, int iterationCount, float scale, Stats sample); /// /// OnMeasure is signaled every time a Measure() is called. /// public event MeasureCallback OnMeasure; public Stats Measure(string name, Action action) { return Measure(name, 1, action, null); } /// /// The main measurment routine. Calling this will cause code:OnMeasure event to be /// raised. /// /// name of the benchmark /// The number of times the benchmark is cloned in 'action' (typically 1) /// The actual code to measure. /// A Stats object representing the measurements (in usec) public Stats Measure(string name, float scale, Action action) { return Measure(name, scale, action, null); } /// /// The main measurment routine. Calling this will cause code:OnMeasure event to be /// raised. /// /// name of the benchmark /// The number of times the benchmark is cloned in 'action' (typically 1) /// The actual code to measure. /// Code that will be called before 'action' to reset the state of the benchmark. /// A Stats object representing the measurements (in usec) public Stats Measure(string name, float scale, Action action, Action reset) { if (reset != null && IterationCount != 1) throw new ApplicationException("Reset can only be used on timers with an iteration count of 1"); Stats statsUSec = new Stats(); if (Prime) { if (reset != null) reset(); action(); } for (int i = 0; i < SampleCount; i++) { if (reset != null) reset(); statsUSec.Add(timer.Measure(name, scale, action)); } if (OnMeasure != null) OnMeasure(name, IterationCount, scale, statsUSec); return statsUSec; } /// /// Prints the mean, median, min, max, and stdDev and count of the samples to the Console /// Useful as a target for OnMeasure /// public static MeasureCallback PrintStats = delegate(string name, int iterationCount, float scale, Stats sample) { Console.WriteLine(name + ": " + sample.ToString()); }; /// /// Prints the mean with a error bound (2 standard deviations, which imply a you have /// 95% confidence that a sampleUsec will be with the bounds (for a normal distribution). /// This is a good default target for OnMeasure. /// public static MeasureCallback Print = delegate(string name, int iterationCount, float scale, Stats sample) { // +- two standard deviations covers 95% of all samples in a normal distribution float errorPercent = (sample.StandardDeviation * 2 * 100) / Math.Abs(sample.Mean); string errorString = ">400%"; if (errorPercent < 400) errorString = (errorPercent.ToString("f0") + "%").PadRight(5); string countString = ""; if (iterationCount != 1) countString = "count: " + iterationCount.ToString() + " "; Console.WriteLine(name + ": " + countString + sample.Mean.ToString("f3").PadLeft(8) + " +- " + errorString + " msec"); }; #region privates CodeTimer timer; #endregion }; /// /// CodeTimer is a simple wrapper that uses System.Diagnostics.StopWatch /// to time the body of some code (given by a delegate), to high precision. /// public class CodeTimer { public CodeTimer() : this(1) { } public CodeTimer(int iterationCount) { this.iterationCount = iterationCount; Prime = true; // Spin the CPU for a while. This should help insure that the CPU gets out of any low power // mode so so that we get more stable results. // TODO: see if this is true, and if there is a better way of doing it. Stopwatch sw = Stopwatch.StartNew(); while (sw.ElapsedMilliseconds < 32) { } } /// /// The number of times the benchmark is run in a loop for a single measument. /// public int IterationCount { get { return iterationCount; } set { iterationCount = value; overheadValid = false; } } /// /// By default CodeTimer will run the action once before doing a /// measurement run. This insures one-time actions like JIT /// compilation are not being measured. However if the benchmark is /// not idempotent, this can be a problem. Setting Prime=false /// insures that this Priming does not happen. /// public bool Prime; public delegate void MeasureCallback(string name, int iterationCount, float sample); /// /// OnMeasure is signaled every time a Measure() is called. /// public event MeasureCallback OnMeasure; /// /// The smallest time (in microseconds) that can be resolved by the timer). /// public static float ResolutionUsec { get { return 1000000.0F / Stopwatch.Frequency; } } /// /// Returns the number of microsecond it took to run 'action', 'count' times. /// public float Measure(string name, Action action) { return Measure(name, 1, action); } /// /// Returns the number of microseconds it to to run action 'count' times divided by 'scale'. /// Scaling is useful if you want to normalize to a single iteration for example. /// public float Measure(string name, float scale, Action action) { Stopwatch sw = new Stopwatch(); // Run the action once to do any JITTing that might happen. if (Prime) action(); float overheadUsec = GetOverheadUsec(action); sw.Reset(); sw.Start(); for (int j = 0; j < iterationCount; j++) action(); sw.Stop(); float sampleUsec = (float)((sw.Elapsed.TotalMilliseconds * 1000.0F - overheadUsec) / scale / iterationCount); if (!computingOverhead && OnMeasure != null) OnMeasure(name, iterationCount, sampleUsec); return sampleUsec; } /// /// Prints the result of a CodeTimer to standard output. /// This is a good default target for OnMeasure. /// public static MeasureCallback Print = delegate(string name, int iterationCount, float sample) { Console.WriteLine("{0}: count={1} time={2:f3} msec ", name, iterationCount, sample); }; #region privates /// /// Time the overheadUsec of the harness that does nothing so we can subtract it out. /// /// Because calling delegates on static methods is more expensive than caling delegates on /// instance methods we need the action to determine the overheadUsec. /// /// float GetOverheadUsec(Action action) { if (!overheadValid) { if (computingOverhead) return 0.0F; computingOverhead = true; // Compute the overheads of calling differnet types of delegates. Action emptyInstanceAction = new Action(this.emptyMethod); // Prime the actions (JIT them) Measure(null, emptyInstanceAction); // Compute the min over 5 runs (figuring better not to go negative) instanceOverheadUsec = float.MaxValue; for (int i = 0; i < 5; i++) { // We multiply by iteration count because we don't want this scaled by the // count but 'Measure' does it by whether we want it or not. instanceOverheadUsec = Math.Min(Measure(null, emptyInstanceAction) * IterationCount, instanceOverheadUsec); } Action emptyStaticAction = new Action(emptyStaticMethod); Measure(null, emptyStaticAction); staticOverheadUsec = float.MaxValue; for (int i = 0; i < 5; i++) staticOverheadUsec = Math.Min(Measure(null, emptyStaticAction) * IterationCount, staticOverheadUsec); computingOverhead = false; overheadValid = true; } if (action.Target == null) return staticOverheadUsec; else return instanceOverheadUsec; } static private void emptyStaticMethod() { } private void emptyMethod() { } bool overheadValid; bool computingOverhead; int iterationCount; float staticOverheadUsec; float instanceOverheadUsec; #endregion }; }