Reporting mechanism much more matured now, and collects usage statistics on several OpenID feature areas.

It also knows how to publish reports.

8 files changed, 384 insertions, 20 deletions

diff --git a/src/DotNetOpenAuth/Messaging/HttpRequestInfo.cs b/src/DotNetOpenAuth/Messaging/HttpRequestInfo.cs
index 2951514..6c7f2f9 100644
--- a/src/DotNetOpenAuth/Messaging/HttpRequestInfo.cs
+++ b/src/DotNetOpenAuth/Messaging/HttpRequestInfo.cs
@@ -73,6 +73,8 @@ namespace DotNetOpenAuth.Messaging {
 			// these as well.
 			this.form = request.Form;
 			this.queryString = request.QueryString;
+
+			Reporting.RecordRequestStatistics(this);
 		}
 
 		/// <summary>
@@ -96,6 +98,8 @@ namespace DotNetOpenAuth.Messaging {
 			this.RawUrl = rawUrl;
 			this.Headers = headers;
 			this.InputStream = inputStream;
+
+			Reporting.RecordRequestStatistics(this);
 		}
 
 		/// <summary>
@@ -115,6 +119,8 @@ namespace DotNetOpenAuth.Messaging {
 			}
 
 			this.InputStream = listenerRequest.InputStream;
+
+			Reporting.RecordRequestStatistics(this);
 		}
 
 		/// <summary>
@@ -131,6 +137,8 @@ namespace DotNetOpenAuth.Messaging {
 			this.Url = requestUri;
 			this.UrlBeforeRewriting = requestUri;
 			this.RawUrl = MakeUpRawUrlFromUrl(requestUri);
+
+			Reporting.RecordRequestStatistics(this);
 		}
 
 		/// <summary>
@@ -157,6 +165,8 @@ namespace DotNetOpenAuth.Messaging {
 			this.RawUrl = MakeUpRawUrlFromUrl(request.RequestUri);
 			this.Headers = GetHeaderCollection(request.Headers);
 			this.InputStream = null;
+
+			Reporting.RecordRequestStatistics(this);
 		}
 
 		/// <summary>
diff --git a/src/DotNetOpenAuth/Messaging/MessagingUtilities.cs b/src/DotNetOpenAuth/Messaging/MessagingUtilities.cs
index e1e3f59..3f77a25 100644
--- a/src/DotNetOpenAuth/Messaging/MessagingUtilities.cs
+++ b/src/DotNetOpenAuth/Messaging/MessagingUtilities.cs
@@ -152,6 +152,7 @@ namespace DotNetOpenAuth.Messaging {
 			Contract.Requires<ArgumentNullException>(requestHandler != null);
 			Contract.Requires<ArgumentNullException>(parts != null);
 
+			Reporting.RecordFeatureUse("MessagingUtilities.PostMultipart");
 			string boundary = Guid.NewGuid().ToString();
 			string partLeadingBoundary = string.Format(CultureInfo.InvariantCulture, "\r\n--{0}\r\n", boundary);
 			string finalTrailingBoundary = string.Format(CultureInfo.InvariantCulture, "\r\n--{0}--\r\n", boundary);
diff --git a/src/DotNetOpenAuth/OpenId/ChannelElements/ExtensionsBindingElement.cs b/src/DotNetOpenAuth/OpenId/ChannelElements/ExtensionsBindingElement.cs
index 84adc59..1c20f1d 100644
--- a/src/DotNetOpenAuth/OpenId/ChannelElements/ExtensionsBindingElement.cs
+++ b/src/DotNetOpenAuth/OpenId/ChannelElements/ExtensionsBindingElement.cs
@@ -88,6 +88,8 @@ namespace DotNetOpenAuth.OpenId.ChannelElements {
 				foreach (IExtensionMessage protocolExtension in extendableMessage.Extensions) {
 					var extension = protocolExtension as IOpenIdMessageExtension;
 					if (extension != null) {
+						Reporting.RecordFeatureUse(protocolExtension);
+
 						// Give extensions that require custom serialization a chance to do their work.
 						var customSerializingExtension = extension as IMessageWithEvents;
 						if (customSerializingExtension != null) {
@@ -145,6 +147,7 @@ namespace DotNetOpenAuth.OpenId.ChannelElements {
 			if (extendableMessage != null) {
 				// First add the extensions that are signed by the Provider.
 				foreach (IOpenIdMessageExtension signedExtension in this.GetExtensions(extendableMessage, true, null)) {
+					Reporting.RecordFeatureUse(signedExtension);
 					signedExtension.IsSignedByRemoteParty = true;
 					extendableMessage.Extensions.Add(signedExtension);
 				}
@@ -154,6 +157,7 @@ namespace DotNetOpenAuth.OpenId.ChannelElements {
 				if (this.relyingPartySecuritySettings == null || !this.relyingPartySecuritySettings.IgnoreUnsignedExtensions) {
 					Func<string, bool> isNotSigned = typeUri => !extendableMessage.Extensions.Cast<IOpenIdMessageExtension>().Any(ext => ext.TypeUri == typeUri);
 					foreach (IOpenIdMessageExtension unsignedExtension in this.GetExtensions(extendableMessage, false, isNotSigned)) {
+						Reporting.RecordFeatureUse(unsignedExtension);
 						unsignedExtension.IsSignedByRemoteParty = false;
 						extendableMessage.Extensions.Add(unsignedExtension);
 					}
diff --git a/src/DotNetOpenAuth/OpenId/Provider/OpenIdProvider.cs b/src/DotNetOpenAuth/OpenId/Provider/OpenIdProvider.cs
index 3eb24d4..4955611 100644
--- a/src/DotNetOpenAuth/OpenId/Provider/OpenIdProvider.cs
+++ b/src/DotNetOpenAuth/OpenId/Provider/OpenIdProvider.cs
@@ -84,6 +84,8 @@ namespace DotNetOpenAuth.OpenId.Provider {
 			}
 
 			this.Channel = new OpenIdChannel(this.AssociationStore, nonceStore, this.SecuritySettings);
+
+			Reporting.RecordFeatureAndDependencyUse(this, associationStore, nonceStore);
 		}
 
 		/// <summary>
diff --git a/src/DotNetOpenAuth/OpenId/RelyingParty/OpenIdRelyingParty.cs b/src/DotNetOpenAuth/OpenId/RelyingParty/OpenIdRelyingParty.cs
index e2d6356..9336f73 100644
--- a/src/DotNetOpenAuth/OpenId/RelyingParty/OpenIdRelyingParty.cs
+++ b/src/DotNetOpenAuth/OpenId/RelyingParty/OpenIdRelyingParty.cs
@@ -104,6 +104,8 @@ namespace DotNetOpenAuth.OpenId.RelyingParty {
 
 			this.channel = new OpenIdChannel(associationStore, nonceStore, this.SecuritySettings);
 			this.AssociationManager = new AssociationManager(this.Channel, associationStore, this.SecuritySettings);
+
+			Reporting.RecordFeatureAndDependencyUse(this, associationStore, nonceStore);
 		}
 
 		/// <summary>
diff --git a/src/DotNetOpenAuth/OpenId/RelyingParty/OpenIdRelyingPartyControlBase.cs b/src/DotNetOpenAuth/OpenId/RelyingParty/OpenIdRelyingPartyControlBase.cs
index 09fcbcb..538e9f2 100644
--- a/src/DotNetOpenAuth/OpenId/RelyingParty/OpenIdRelyingPartyControlBase.cs
+++ b/src/DotNetOpenAuth/OpenId/RelyingParty/OpenIdRelyingPartyControlBase.cs
@@ -244,6 +244,7 @@ namespace DotNetOpenAuth.OpenId.RelyingParty {
 		/// Initializes a new instance of the <see cref="OpenIdRelyingPartyControlBase"/> class.
 		/// </summary>
 		protected OpenIdRelyingPartyControlBase() {
+			Reporting.RecordFeatureUse(this);
 		}
 
 		#region Events
diff --git a/src/DotNetOpenAuth/OpenId/RelyingParty/PositiveAuthenticationResponse.cs b/src/DotNetOpenAuth/OpenId/RelyingParty/PositiveAuthenticationResponse.cs
index d4d0cdc..e809205 100644
--- a/src/DotNetOpenAuth/OpenId/RelyingParty/PositiveAuthenticationResponse.cs
+++ b/src/DotNetOpenAuth/OpenId/RelyingParty/PositiveAuthenticationResponse.cs
@@ -37,7 +37,6 @@ namespace DotNetOpenAuth.OpenId.RelyingParty {
 				null);
 
 			this.VerifyDiscoveryMatchesAssertion(relyingParty);
-			Reporting.OnAuthenticated();
 		}
 
 		#region IAuthenticationResponse Properties
diff --git a/src/DotNetOpenAuth/Reporting.cs b/src/DotNetOpenAuth/Reporting.cs
index 209a128..51b8b1f 100644
--- a/src/DotNetOpenAuth/Reporting.cs
+++ b/src/DotNetOpenAuth/Reporting.cs
@@ -8,12 +8,18 @@ namespace DotNetOpenAuth {
 	using System;
 	using System.Collections.Generic;
 	using System.Diagnostics;
+	using System.Diagnostics.Contracts;
+	using System.Globalization;
 	using System.IO;
 	using System.IO.IsolatedStorage;
 	using System.Linq;
+	using System.Net;
 	using System.Reflection;
 	using System.Text;
+	using System.Threading;
+	using System.Web;
 	using DotNetOpenAuth.Configuration;
+	using DotNetOpenAuth.Messaging;
 
 	/// <summary>
 	/// The statistical reporting mechanism used so this library's project authors
@@ -21,9 +27,59 @@ namespace DotNetOpenAuth {
 	/// </summary>
 	internal static class Reporting {
 		/// <summary>
-		/// The writer to use to log statistics.
+		/// The maximum frequency that reports will be published.
 		/// </summary>
-		private static StreamWriter writer;
+		private static readonly TimeSpan minimumReportingInterval = TimeSpan.FromDays(1);
+
+		/// <summary>
+		/// The isolated storage to use for collecting data in between published reports.
+		/// </summary>
+		private static IsolatedStorageFile file;
+
+		/// <summary>
+		/// The name of this assembly.
+		/// </summary>
+		private static AssemblyName assemblyName;
+
+		/// <summary>
+		/// The recipient of collected reports.
+		/// </summary>
+		private static Uri wellKnownPostLocation = new Uri("http://reports.dotnetopenauth.net/ReportingPost.aspx");
+
+		/// <summary>
+		/// The outgoing HTTP request handler to use for publishing reports.
+		/// </summary>
+		private static IDirectWebRequestHandler webRequestHandler;
+
+		/// <summary>
+		/// A few HTTP request hosts and paths we've seen.
+		/// </summary>
+		private static PersistentHashSet observedRequests;
+
+		/// <summary>
+		/// Cultures that have come in via HTTP requests.
+		/// </summary>
+		private static PersistentHashSet observedCultures;
+
+		/// <summary>
+		/// Features that have been used.
+		/// </summary>
+		private static PersistentHashSet observedFeatures;
+
+		/// <summary>
+		/// A collection of all the observations to include in the report.
+		/// </summary>
+		private static List<PersistentHashSet> observations = new List<PersistentHashSet>();
+
+		/// <summary>
+		/// The lock acquired while considering whether to publish a report.
+		/// </summary>
+		private static object publishingConsiderationLock = new object();
+
+		/// <summary>
+		/// The time that we last published reports.
+		/// </summary>
+		private static DateTime lastPublished = DateTime.Now;
 
 		/// <summary>
 		/// Initializes static members of the <see cref="Reporting"/> class.
@@ -32,9 +88,20 @@ namespace DotNetOpenAuth {
 			Enabled = DotNetOpenAuthSection.Configuration.Reporting.Enabled;
 			if (Enabled) {
 				try {
-					writer = OpenReport();
-					writer.WriteLine();
-					writer.WriteLine(Util.LibraryVersion);
+					file = IsolatedStorageFile.GetUserStoreForDomain();
+					assemblyName = new AssemblyName(Assembly.GetExecutingAssembly().FullName);
+					webRequestHandler = new StandardWebRequestHandler();
+					observations.Add(observedRequests = new PersistentHashSet(file, "requests.txt", 3));
+					observations.Add(observedCultures = new PersistentHashSet(file, "cultures.txt", 20));
+					observations.Add(observedFeatures = new PersistentHashSet(file, "features.txt", int.MaxValue));
+
+					// Record site-wide features in use.
+					if (HttpContext.Current != null && HttpContext.Current.ApplicationInstance != null) {
+						// MVC or web forms?
+						// front-end or back end web farm?
+						// url rewriting?
+						////RecordFeatureUse(IsMVC ? "ASP.NET MVC" : "ASP.NET Web Forms");
+					}
 				} catch {
 					// This is supposed to be as low-risk as possible, so if it fails, just disable reporting.
 					Enabled = false;
@@ -49,27 +116,305 @@ namespace DotNetOpenAuth {
 		private static bool Enabled { get; set; }
 
 		/// <summary>
-		/// Called when an OpenID RP successfully authenticates someone.
+		/// Records the use of a feature by name.
 		/// </summary>
-		internal static void OnAuthenticated() {
-			if (!Enabled) {
-				return;
+		/// <param name="feature">The feature.</param>
+		internal static void RecordFeatureUse(string feature) {
+			Contract.Requires<ArgumentException>(!String.IsNullOrEmpty(feature));
+
+			if (Enabled) {
+				observedFeatures.Add(feature);
+				Touch();
 			}
+		}
+
+		/// <summary>
+		/// Records the use of a feature by object type.
+		/// </summary>
+		/// <param name="value">The object whose type is the feature to set as used.</param>
+		internal static void RecordFeatureUse(object value) {
+			Contract.Requires<ArgumentNullException>(value != null);
 
-			writer.Write("L");
+			if (Enabled) {
+				observedFeatures.Add(value.GetType().Name);
+				Touch();
+			}
 		}
 
 		/// <summary>
-		/// Opens the report file for append.
+		/// Records the use of a feature by object type.
 		/// </summary>
-		/// <returns>The writer to use.</returns>
-		private static StreamWriter OpenReport() {
-			IsolatedStorageFile file = IsolatedStorageFile.GetUserStoreForDomain();
-			var assemblyName = new AssemblyName(Assembly.GetExecutingAssembly().FullName);
-			var fileStream = new IsolatedStorageFileStream("reporting.txt", FileMode.Append, FileAccess.Write, FileShare.Read);
-			var writer = new StreamWriter(fileStream, Encoding.UTF8);
-			writer.AutoFlush = true;
-			return writer;
+		/// <param name="value">The object whose type is the feature to set as used.</param>
+		/// <param name="dependency1">Some dependency used by <paramref name="value"/>.</param>
+		/// <param name="dependency2">Some dependency used by <paramref name="value"/>.</param>
+		internal static void RecordFeatureAndDependencyUse(object value, object dependency1, object dependency2) {
+			Contract.Requires<ArgumentNullException>(value != null);
+
+			if (Enabled) {
+				StringBuilder builder = new StringBuilder();
+				builder.Append(value.GetType().Name);
+				builder.Append(" ");
+				builder.Append(dependency1 != null ? dependency1.GetType().Name : "(null)");
+				builder.Append(" ");
+				builder.Append(dependency2 != null ? dependency2.GetType().Name : "(null)");
+				observedFeatures.Add(builder.ToString());
+				Touch();
+			}
+		}
+
+		/// <summary>
+		/// Records statistics collected from incoming requests.
+		/// </summary>
+		/// <param name="request">The request.</param>
+		internal static void RecordRequestStatistics(HttpRequestInfo request) {
+			Contract.Requires<ArgumentNullException>(request != null);
+
+			if (Enabled) {
+				observedCultures.Add(Thread.CurrentThread.CurrentCulture.Name);
+
+				if (!observedRequests.IsFull) {
+					var requestBuilder = new UriBuilder(request.UrlBeforeRewriting);
+					requestBuilder.Query = null;
+					requestBuilder.Fragment = null;
+					observedRequests.Add(requestBuilder.Uri.AbsoluteUri);
+				}
+
+				Touch();
+			}
+		}
+
+		/// <summary>
+		/// Assembles a report for submission.
+		/// </summary>
+		/// <returns>A stream that contains the report.</returns>
+		private static Stream GetReport() {
+			var stream = new MemoryStream();
+			var writer = new StreamWriter(stream, Encoding.UTF8);
+			writer.WriteLine(Util.LibraryVersion);
+
+			foreach (var observation in observations) {
+				writer.WriteLine("====================================");
+				writer.WriteLine(observation.FileName);
+				try {
+					using (var fileStream = new IsolatedStorageFileStream(observation.FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, file)) {
+						writer.Flush();
+						fileStream.CopyTo(writer.BaseStream);
+					}
+				} catch (FileNotFoundException) {
+					writer.WriteLine("(missing)");
+				}
+			}
+
+			// Make sure the stream is positioned at the beginning.
+			writer.Flush();
+			stream.Position = 0;
+			return stream;
+		}
+
+		/// <summary>
+		/// Sends the usage reports to the library authors.
+		/// </summary>
+		/// <returns>A value indicating whether submitting the report was successful.</returns>
+		private static bool SendStats() {
+			try {
+				var request = (HttpWebRequest)WebRequest.Create(wellKnownPostLocation);
+				request.UserAgent = Util.LibraryVersion;
+				request.AllowAutoRedirect = false;
+				request.Method = "POST";
+				var report = GetReport();
+				request.ContentLength = report.Length;
+				using (var requestStream = webRequestHandler.GetRequestStream(request)) {
+					report.CopyTo(requestStream);
+				}
+
+				var response = webRequestHandler.GetResponse(request);
+				response.Dispose();
+				return true;
+			} catch (ProtocolException ex) {
+				Logger.Library.Error("Unable to submit report due to an HTTP error.", ex);
+			} catch (FileNotFoundException ex) {
+				Logger.Library.Error("Unable to submit report because the report file is missing.", ex);
+			}
+
+			return false;
+		}
+
+		/// <summary>
+		/// Called by every internal/public method on this class to give
+		/// periodic operations a chance to run.
+		/// </summary>
+		private static void Touch() {
+			// Publish stats if it's time to do so.
+			lock (publishingConsiderationLock) {
+				if (DateTime.Now - lastPublished > minimumReportingInterval) {
+					lastPublished = DateTime.Now;
+
+					// Do it on a background thread since it could take a while and we
+					// don't want to slow down this request we're borrowing.
+					ThreadPool.QueueUserWorkItem(state => SendStats());
+				}
+			}
+		}
+
+		/// <summary>
+		/// A set of values that persist the set to disk.
+		/// </summary>
+		private class PersistentHashSet : IDisposable {
+			/// <summary>
+			/// The isolated persistent storage.
+			/// </summary>
+			private readonly FileStream fileStream;
+
+			/// <summary>
+			/// The persistent reader.
+			/// </summary>
+			private readonly StreamReader reader;
+
+			/// <summary>
+			/// The persistent writer.
+			/// </summary>
+			private readonly StreamWriter writer;
+
+			/// <summary>
+			/// The total set of elements.
+			/// </summary>
+			private readonly HashSet<string> memorySet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+			/// <summary>
+			/// The maximum frequency the set can be flushed to disk.
+			/// </summary>
+#if DEBUG
+			private static readonly TimeSpan minimumFlushInterval = TimeSpan.FromSeconds(30);
+#else
+			private static readonly TimeSpan minimumFlushInterval = TimeSpan.FromMinutes(15);
+#endif
+
+			/// <summary>
+			/// The maximum number of elements to track before not storing new elements.
+			/// </summary>
+			private readonly int maximumElements;
+
+			/// <summary>
+			/// The set of new elements added to the <see cref="memorySet"/> since the last flush.
+			/// </summary>
+			private List<string> newElements = new List<string>();
+
+			/// <summary>
+			/// The time the last flush occurred.
+			/// </summary>
+			private DateTime lastFlushed;
+
+			/// <summary>
+			/// A flag indicating whether the set has changed since it was last flushed.
+			/// </summary>
+			private bool dirty;
+
+			/// <summary>
+			/// Initializes a new instance of the <see cref="PersistentHashSet"/> class.
+			/// </summary>
+			/// <param name="storage">The storage location.</param>
+			/// <param name="fileName">Name of the file.</param>
+			/// <param name="maximumElements">The maximum number of elements to track.</param>
+			internal PersistentHashSet(IsolatedStorageFile storage, string fileName, int maximumElements) {
+				Contract.Requires<ArgumentNullException>(storage != null);
+				Contract.Requires<ArgumentException>(!String.IsNullOrEmpty(fileName));
+				this.FileName = fileName;
+				this.maximumElements = maximumElements;
+
+				// Load the file into memory.
+				bool fileCreated = storage.GetFileNames(fileName).Length == 0;
+				this.fileStream = new IsolatedStorageFileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, storage);
+				this.reader = new StreamReader(this.fileStream, Encoding.UTF8);
+				while (!this.reader.EndOfStream) {
+					this.memorySet.Add(this.reader.ReadLine());
+				}
+
+				this.writer = new StreamWriter(this.fileStream, Encoding.UTF8);
+				this.lastFlushed = DateTime.Now;
+
+				// Write a unique header to the file so the report collector can match duplicates.
+				if (fileCreated) {
+					this.writer.WriteLine(Guid.NewGuid());
+				}
+			}
+
+			/// <summary>
+			/// Gets a value indicating whether the hashset has reached capacity and is not storing more elements.
+			/// </summary>
+			/// <value><c>true</c> if this instance is full; otherwise, <c>false</c>.</value>
+			internal bool IsFull {
+				get {
+					lock (this.memorySet) {
+						return this.memorySet.Count >= this.maximumElements;
+					}
+				}
+			}
+
+			internal string FileName { get; private set; }
+
+			#region IDisposable Members
+
+			/// <summary>
+			/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+			/// </summary>
+			public void Dispose() {
+				this.Dispose(true);
+			}
+
+			#endregion
+
+			/// <summary>
+			/// Adds a value to the set.
+			/// </summary>
+			/// <param name="value">The value.</param>
+			internal void Add(string value) {
+				lock (this.memorySet) {
+					if (!this.IsFull) {
+						if (this.memorySet.Add(value)) {
+							this.newElements.Add(value);
+							this.dirty = true;
+
+							if (this.IsFull) {
+								this.Flush();
+							}
+						}
+
+						if (this.dirty && DateTime.Now - this.lastFlushed > minimumFlushInterval) {
+							this.Flush();
+						}
+					}
+				}
+			}
+
+			/// <summary>
+			/// Flushes any newly added values to disk.
+			/// </summary>
+			internal void Flush() {
+				lock (this.memorySet) {
+					foreach (string element in this.newElements) {
+						this.writer.WriteLine(element);
+					}
+					this.writer.Flush();
+
+					// Assign a whole new list since future lists might be smaller in order to
+					// decrease demand on memory.
+					this.newElements = new List<string>();
+					this.dirty = false;
+					this.lastFlushed = DateTime.Now;
+				}
+			}
+
+			/// <summary>
+			/// Releases unmanaged and - optionally - managed resources
+			/// </summary>
+			/// <param name="disposing"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
+			protected virtual void Dispose(bool disposing) {
+				if (disposing) {
+					this.writer.Dispose();
+					this.reader.Dispose();
+					this.fileStream.Dispose();
+				}
+			}
 		}
 	}
 }