path: root/tools/Sandcastle/Source/BuildAssembler/BuildComponents/SnippetComponent.cs

// Copyright © Microsoft Corporation.
// This source file is subject to the Microsoft Permissive License.
// See http://www.microsoft.com/resources/sharedsource/licensingbasics/sharedsourcelicenses.mspx.
// All other rights reserved.

// <summary>Contains code to insert snippets directly from the source files without using any 
// intermediate XML files.
// </summary>
namespace Microsoft.Ddue.Tools
{
    using System;
    using System.Configuration;
    using System.Collections.Generic;
    using System.IO;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Xml;
    using System.Xml.XPath;
    using System.Globalization;
    using System.Diagnostics;

    /// <summary>
    /// SnippetComponent class to replace the snippet code references.
    /// </summary>
    public class SnippetComponent : BuildComponent
    {
        #region Private members
        /// <summary>
        /// Regex to validate the snippet references.
        /// </summary>
        private static Regex validSnippetReference = new Regex(
            @"^[^#\a\b\f\n\r\t\v]+#(\w+,)*\w+$",
            RegexOptions.Compiled);

        /// <summary>
        /// Dictionary to map language folder names to language id.
        /// </summary>
        private static Dictionary<string, string> languageMap = new Dictionary<string, string>(StringComparer.CurrentCultureIgnoreCase);

        /// <summary>
        /// List that controls the order in which languages snippets are displayed.
        /// </summary>
        private static List<string> languageList = new List<string>();

        /// <summary>
        /// Dictionary consisting of example name as key and example path as value.
        /// </summary>
        private Dictionary<string, string> exampleIndex = new Dictionary<string, string>();

        /// <summary>
        /// Dictionary consisting of exampleName\unitName as key with a null value.
        /// </summary>
        private Dictionary<string, string> approvedSnippetIndex = new Dictionary<string,string>();

        /// <summary>
        /// Dictionary containing the example name as key and list of rejected language snippets as values.
        /// </summary>
        private Dictionary<string, List<string>> rejectedSnippetIndex = new Dictionary<string, List<string>>();

        /// <summary>
        /// List of unit folder names to exclude from sample parsing.
        /// </summary>
        private Dictionary<string, Object> excludedUnits = new Dictionary<string, Object>();

        /// <summary>
        /// Dictionary consisting of exampleName\unitName as key with a null value.
        /// </summary>
        private SnippetCache snippetCache = null;

        /// <summary>
        /// XPathExpression to look for snippet references in the topics.
        /// </summary>
        private XPathExpression selector;

        /// <summary>
        /// XmlNamespaceManager to set the context.
        /// </summary>
        private XmlNamespaceManager context = new CustomContext();

        /// <summary>
        /// List of languages.
        /// </summary>
        private List<Language> languages = new List<Language>();

        /// <summary>
        /// snippet store.
        /// </summary>
        private Dictionary<SnippetIdentifier, List<Snippet>> snippets = new Dictionary<SnippetIdentifier, List<Snippet>>();
        #endregion

        #region Constructor

        /// <summary>
        /// Constructor for SnippetComponent class.
        /// </summary>
        /// <param name="assembler">An instance of Build Assembler</param>
        /// <param name="configuration">configuration to be parsed for information related to snippets</param>
        public SnippetComponent(BuildAssembler assembler, XPathNavigator configuration)
            : base(assembler, configuration)
        {
            Debug.Assert(assembler != null);
            Debug.Assert(configuration != null);

            // Get the parsnip examples location.
            XPathNodeIterator examplesNode = configuration.Select("examples/example");

            if (examplesNode.Count == 0)
                WriteMessage(MessageLevel.Error, "Each snippet component element must have a child element named 'examples' containing an element named 'example' with an attribute named 'directory', whose value is a path to the directory containing examples.");

            foreach (XPathNavigator exampleNode in examplesNode)
            {
                string rootDirectory = exampleNode.GetAttribute("directory", string.Empty);

                if (string.IsNullOrEmpty(rootDirectory))
                    WriteMessage(MessageLevel.Error, "Each examples element must have a directory attribute specifying a directory containing parsnip samples.");

                rootDirectory = Environment.ExpandEnvironmentVariables(rootDirectory);
                if (!Directory.Exists(rootDirectory))
                    WriteMessage(MessageLevel.Error, String.Format("The examples/@directory attribute specified a directory that doesn't exist: '{0}'", rootDirectory));

                // create a dictionary that maps the example names to the example path under the root directory
                this.loadExamples(rootDirectory);
            }

            // Get the approved log files location.
            XPathNodeIterator approvedSnippetsNode = configuration.Select("approvalLogs/approvalLog");

            if (approvedSnippetsNode.Count == 0)
                WriteMessage(MessageLevel.Warn, "The config did not have an 'approvalLogs' node to specify a snippet approval logs.");
            
            foreach (XPathNavigator node in approvedSnippetsNode)
            {
                string approvalLogFile = node.GetAttribute("file", string.Empty);

                if (string.IsNullOrEmpty(approvalLogFile))
                    WriteMessage(MessageLevel.Error, "The approvalLog node must have a 'file' attribute specifying the path to a snippet approval log.");

                approvalLogFile = Environment.ExpandEnvironmentVariables(approvalLogFile);
                if (!File.Exists(approvalLogFile))
                    WriteMessage(MessageLevel.Error, String.Format("The approvalLog/@file attribute specified a file that doesn't exist: '{0}'", approvalLogFile));

                // load the approval log into the approvedSnippetIndex dictionary
                this.parseApprovalLogFiles(approvalLogFile);
            }

            // Get the names of the unit directories in the sample tree to exclude from parsing
            //     <excludedUnits><unitFolder name="CPP_OLD" /></excludedUnits>
            XPathNodeIterator excludedUnitNodes = configuration.Select("excludedUnits/unitFolder");
            foreach (XPathNavigator unitFolder in excludedUnitNodes)
            {
                string folderName = unitFolder.GetAttribute("name", string.Empty);

                if (string.IsNullOrEmpty(folderName))
                    WriteMessage(MessageLevel.Error, "Each excludedUnits/unitFolder node must have a 'name' attribute specifying the name of a folder name to exclude.");

                folderName = Environment.ExpandEnvironmentVariables(folderName);

                // add the folderName to the list of names to be excluded
                this.excludedUnits.Add(folderName.ToLower(),null);
            }

            // Get the languages defined.
            XPathNodeIterator languageNodes = configuration.Select("languages/language");
            foreach (XPathNavigator languageNode in languageNodes)
            {
                // read the @languageId, @unit, and @extension attributes
                string languageId = languageNode.GetAttribute("languageId", string.Empty);
                if (string.IsNullOrEmpty(languageId))
                    WriteMessage(MessageLevel.Error, "Each language node must specify an @languageId attribute.");

                string unit = languageNode.GetAttribute("unit", string.Empty);

                // if both @languageId and @unit are specified, add this language to the language map
                if (!string.IsNullOrEmpty(unit))
                    languageMap.Add(unit.ToLower(), languageId);

                // add languageId to the languageList for purpose of ordering snippets in the output
                if (!languageList.Contains(languageId))
                    languageList.Add(languageId.ToLower());

                string extension = languageNode.GetAttribute("extension", string.Empty);
                if (!string.IsNullOrEmpty(extension))
                {
                    if (!extension.Contains("."))
                    {
                        extension = "." + extension;
                        WriteMessage(MessageLevel.Warn, String.Format("The @extension value must begin with a period. Adding a period to the extension value '{0}' of the {1} language.", extension, languageId));
                    }
                    else
                    {
                        int indexOfPeriod = extension.IndexOf('.');
                        if (indexOfPeriod != 0)
                        {
                            extension = extension.Substring(indexOfPeriod);
                            WriteMessage(MessageLevel.Warn, String.Format("The @extension value must begin with a period. Using the substring beginning with the first period of the specified extension value '{0}' of the {1} language.", extension, languageId));
                        }
                    }
                }

                // read the color nodes, if any, and add them to the list of colorization rules
                List<ColorizationRule> rules = new List<ColorizationRule>();

                XPathNodeIterator colorNodes = languageNode.Select("color");
                foreach (XPathNavigator colorNode in colorNodes)
                {
                    string pattern = colorNode.GetAttribute("pattern", String.Empty);
                    string region = colorNode.GetAttribute("region", String.Empty);
                    string name = colorNode.GetAttribute("class", String.Empty);
                    if (String.IsNullOrEmpty(region))
                    {
                        rules.Add(new ColorizationRule(pattern, name));
                    }
                    else
                    {
                        rules.Add(new ColorizationRule(pattern, region, name));
                    }
                }

                this.languages.Add(new Language(languageId, extension, rules));
                WriteMessage(MessageLevel.Info, String.Format("Loaded {0} colorization rules for the language '{1}', extension '{2}.", rules.Count, languageId, extension));
            }

            this.context.AddNamespace("ddue", "http://ddue.schemas.microsoft.com/authoring/2003/5");
            this.selector = XPathExpression.Compile("//ddue:codeReference");
            this.selector.SetContext(this.context);

            // create the snippet cache
            snippetCache = new SnippetCache(100, approvedSnippetIndex, languageMap, languages, excludedUnits);
        }
        #endregion

        #region Public methods
        /// <summary>
        /// Apply method to perform the actual work of the component.
        /// </summary>
        /// <param name="document">document to be parsed for snippet references</param>
        /// <param name="key">Id of a topic</param>
        public override void Apply(XmlDocument document, string key)
        {
            // clear out the snippets dictionary of any snippets from the previous document
            snippets.Clear();

            XPathNodeIterator nodesIterator = document.CreateNavigator().Select(this.selector);
            XPathNavigator[] nodes = BuildComponentUtilities.ConvertNodeIteratorToArray(nodesIterator);
            foreach (XPathNavigator node in nodes)
            {
                // get the snippet reference, which can contain one or more snippet ids
                string reference = node.Value;

                // check for validity of reference
                if (!validSnippetReference.IsMatch(reference))
                {
                    WriteMessage(MessageLevel.Warn, "Skipping invalid snippet reference: " + reference);
                    continue;
                }

                // get the identifiers from the codeReference
                SnippetIdentifier[] identifiers = SnippetIdentifier.ParseReference(reference);

                // load the language-specific snippets for each of the specified identifiers
                foreach (SnippetIdentifier identifier in identifiers)
                {
                    if (snippets.ContainsKey(identifier))
                        continue;

                    // look up the snippets example path
                    string examplePath = string.Empty;
                    if (!this.exampleIndex.TryGetValue(identifier.Example, out examplePath))
                    {
                        WriteMessage(MessageLevel.Warn, String.Format("Snippet with identifier '{0}' was not found. The '{1}' example was not found in the examples directory.", identifier.ToString(), identifier.Example));
                        continue;
                    }

                    // get the snippet from the snippet cache 
                    List<Snippet> snippetList = snippetCache.GetContent(examplePath, identifier);
                    if (snippetList != null)
                    {
                        snippets.Add(identifier, snippetList);
                    }
                    else 
                    {
                        // if no approval log was specified in the config, all snippets are treated as approved by default
                        // so show an warning message that the snippet was not found
                        if (approvedSnippetIndex.Count == 0)
                            WriteMessage(MessageLevel.Warn, string.Format("No Snippet with identifier '{0}' was found.", identifier.ToString()));
                        else
                        {
                            // show a warning message: either snippet not found, or snippet not approved.
                            bool isApproved = false;

                            foreach (string snippetIndex in this.approvedSnippetIndex.Keys)
                            {
                                string[] splitSnippet = snippetIndex.Split('\\');
                                if (splitSnippet[0] == identifier.Example)
                                {
                                    isApproved = true;
                                    break;
                                }
                            }

                            // check whether snippets are present in parsnip approval logs and throw warnings accordingly.
                            if (!isApproved || !rejectedSnippetIndex.ContainsKey(identifier.Example))
                                WriteMessage(MessageLevel.Warn, string.Format("The snippet with identifier '{0}' was omitted because it is not present in parsnip approval logs.", identifier.ToString()));
                            else
                                WriteMessage(MessageLevel.Warn, string.Format("No Snippet with identifier '{0}' was found.", identifier.ToString()));
                        }

                        continue;
                    }

                    // write warning messages for any rejected units for this example
                    List<string> rejectedUnits;
                    if (rejectedSnippetIndex.TryGetValue(identifier.Example, out rejectedUnits))
                    {
                        foreach (string rejectedUnit in rejectedUnits)
                            WriteMessage(MessageLevel.Warn, string.Format("The '{0}' snippet with identifier '{1}' was omitted because the {2}\\{0} unit did not pass Parsnip testing.", rejectedUnit, identifier.ToString(), identifier.Example));
                    }
                }

                if (identifiers.Length == 1)
                {
                    // one snippet referenced
                    SnippetIdentifier identifier = identifiers[0];

                    if (snippets.ContainsKey(identifier))
                    {
                        writeSnippetContent(node, identifier, snippets[identifier]);
                    }
                }
                else
                {
                    // handle case where codeReference contains multiple identifiers
                    // Each language's set of snippets from multiple identifiers are displayed in a single block; 

                    // create dictionary that maps each language to its set of snippets
                    Dictionary<string, List<Snippet>> map = new Dictionary<string, List<Snippet>>();
                    foreach (SnippetIdentifier identifier in identifiers)
                    {
                        List<Snippet> values;
                        if (snippets.TryGetValue(identifier, out values))
                        {
                            foreach (Snippet value in values)
                            {
                                List<Snippet> pieces;
                                if (!map.TryGetValue(value.Language.LanguageId, out pieces))
                                {
                                    pieces = new List<Snippet>();
                                    map.Add(value.Language.LanguageId, pieces);
                                }
                                pieces.Add(value);
                            }
                        }
                    }

                    // now write the collection of snippet pieces to the document
                    XmlWriter writer = node.InsertAfter();
                    writer.WriteStartElement("snippets");
                    writer.WriteAttributeString("reference", reference);

                    // first write the snippets in the order their language shows up in the language map (if any)
                    foreach (string devlang in languageList)
                    {
                        foreach (KeyValuePair<string, List<Snippet>> entry in map)
                        {
                            if (!(devlang == entry.Key.ToLower()))
                                continue;
                            writer.WriteStartElement("snippet");
                            writer.WriteAttributeString("language", entry.Key);
                            writer.WriteString("\n");

                            // write the set of snippets for this language
                            List<Snippet> values = entry.Value;
                            for (int i = 0; i < values.Count; i++)
                            {
                                if (i > 0)
                                    writer.WriteString("\n...\n\n\n");
                                // write the colorized or plaintext snippet text
                                WriteSnippetText(values[i], writer);
                            }
                            
                            writer.WriteEndElement();
                        }
                    }

                    // now write any snippets whose language isn't in the language map
                    foreach (KeyValuePair<string, List<Snippet>> entry in map)
                    {
                        if (languageList.Contains(entry.Key.ToLower()))
                            continue;
                        writer.WriteStartElement("snippet");
                        writer.WriteAttributeString("language", entry.Key);
                        writer.WriteString("\n");

                        // write the set of snippets for this language
                        List<Snippet> values = entry.Value;
                        for (int i = 0; i < values.Count; i++)
                        {
                            if (i > 0)
                                writer.WriteString("\n...\n\n\n");
                            // write the colorized or plaintext snippet text
                            WriteSnippetText(values[i], writer);
                        }
                        
                        writer.WriteEndElement();
                    }
                    
                    writer.WriteEndElement();
                    writer.Close();
                }
                node.DeleteSelf();
            }
        }
        #endregion

        #region Private methods
        /// <summary>
        /// Index the example names to paths.
        /// </summary>
        /// <param name="rootDirectory">root directory location of parsnip samples</param>
        private void loadExamples(string rootDirectory)
        {
            try
            {
                DirectoryInfo root = new DirectoryInfo(rootDirectory);
                DirectoryInfo[] areaDirectories = root.GetDirectories();

                foreach (DirectoryInfo area in areaDirectories)
                {
                    DirectoryInfo[] exampleDirectories = area.GetDirectories();

                    foreach (DirectoryInfo example in exampleDirectories)
                    {
                        string path;
                        if (this.exampleIndex.TryGetValue(example.Name.ToLower(CultureInfo.InvariantCulture), out path))
                            WriteMessage(MessageLevel.Warn, string.Format("The example '{0}' under folder '{1}' already exists under '{2}'", example.Name, example.FullName, path));
                        
                        this.exampleIndex[example.Name.ToLower(CultureInfo.InvariantCulture)] = example.FullName;
                    }
                }
            }
            catch (Exception e)
            {
                WriteMessage(MessageLevel.Error, string.Format(System.Threading.Thread.CurrentThread.CurrentCulture, "The loading of examples failed:{0}", e.Message));
                throw;
            }
        }

        /// <summary>
        /// Index the approved snippets.
        /// </summary>
        /// <param name="file">approved snippets log file</param>
        private void parseApprovalLogFiles(string file)
        {
            string sampleName = string.Empty;
            string unitName = string.Empty;
            List<string> rejectedUnits = null;
           
            XmlReader reader = XmlReader.Create(file);
            try
            {
                while (reader.Read())
                {
                    if (reader.NodeType == XmlNodeType.Element)
                    {
                        if (reader.Name == "Sample")
                        {
                           sampleName = reader.GetAttribute("name").ToLower(CultureInfo.InvariantCulture);
                           //create a new rejectedUnits list for this sample
                           rejectedUnits = null;
                        }

                        if (reader.Name == "Unit")
                        {
                            unitName = reader.GetAttribute("name").ToLower(CultureInfo.InvariantCulture);

                            bool include = Convert.ToBoolean(reader.GetAttribute("include"));

                            if (include)
                            {
                               if (this.approvedSnippetIndex.ContainsKey(Path.Combine(sampleName, unitName)))
                                    WriteMessage(MessageLevel.Warn, string.Format("Sample '{0}' already exists in the approval log files.", sampleName));
                                this.approvedSnippetIndex[Path.Combine(sampleName, unitName)] = null;
                            }
                            else
                            {
                                if (rejectedUnits == null)
                                {
                                    rejectedUnits = new List<string>();
                                    rejectedSnippetIndex[sampleName] = rejectedUnits;
                                }
                                rejectedUnits.Add(unitName);
                            }
                        }
                    }
                }
            }
            catch (XmlException e)
            {
                WriteMessage(MessageLevel.Error, String.Format("The specified approval log file is not well-formed. The error message is: {0}", e.Message));
            }
            finally
            {
                reader.Close();
            }
        }

        /// <summary>
        /// Write the snippet content to output files.
        /// </summary>
        /// <param name="node">code reference node</param>
        /// <param name="identifier">List of snippets</param>
        private void writeSnippetContent(XPathNavigator node, SnippetIdentifier identifier, List<Snippet> snippetList)
        {
            if (snippetList == null || snippetList.Count == 0)
            {
                WriteMessage(MessageLevel.Warn, "Empty snippet list past for node " + node.Name);
                return;
            }

            XmlWriter writer = node.InsertAfter();
            writer.WriteStartElement("snippets");
            writer.WriteAttributeString("reference", node.Value);

            // first write the snippets in the order their language shows up in the language map (if any)
            foreach (string devlang in languageList)
            {
                foreach (Snippet snippet in snippetList)
                {
                    if (!(devlang == snippet.Language.LanguageId.ToLower()))
                        continue;
                    writer.WriteStartElement("snippet");
                    writer.WriteAttributeString("language", snippet.Language.LanguageId);
                    writer.WriteString("\n");
                    // write the colorized or plaintext snippet text
                    WriteSnippetText(snippet, writer);
                    writer.WriteEndElement();
                }
            }

            // now write any snippets whose language isn't in the language map
            foreach (Snippet snippet in snippetList)
            {
                if (languageList.Contains(snippet.Language.LanguageId.ToLower()))
                    continue;
                writer.WriteStartElement("snippet");
                writer.WriteAttributeString("language", snippet.Language.LanguageId);
                writer.WriteString("\n");
                // write the colorized or plaintext snippet text
                WriteSnippetText(snippet, writer);
                writer.WriteEndElement();
            }
            
            writer.WriteEndElement();
            writer.Close();
        }

        private void WriteSnippetText(Snippet snippet, XmlWriter writer)
        {
            // if colorization rules are defined, then colorize the snippet.
            if (snippet.Language.ColorizationRules != null)
            {
                writeColorizedSnippet(colorizeSnippet(snippet.Content, snippet.Language.ColorizationRules), writer);
            }
            else
            {
                writer.WriteString(snippet.Content);
            }
        }

        private static ICollection<Region> colorizeSnippet(string text, List<ColorizationRule> rules)
        {
            // Console.WriteLine("colorizing: '{0}'", text);
            // create a linked list consiting entirely of one uncolored region
            LinkedList<Region> regions = new LinkedList<Region>();
            regions.AddFirst(new Region(text));

            // loop over colorization rules
            foreach (ColorizationRule rule in rules)
            {
                // loop over regions
                LinkedListNode<Region> node = regions.First;
                while (node != null)
                {
                    // only try to colorize uncolored regions
                    if (node.Value.ClassName != null)
                    {
                        node = node.Next;
                        continue;
                    }

                    // find matches in the region
                    string regionText = node.Value.Text;
                    Capture[] matches = rule.Apply(regionText);

                    // if no matches were found, continue to the next region
                    if (matches.Length == 0)
                    {
                        node = node.Next;
                        continue;
                    }

                    // we found matches; break the region into colored and uncolered subregions
                    // index is where we are looking from; index-1 is the end of the last match
                    int index = 0;

                    LinkedListNode<Region> referenceNode = node;

                    foreach (Capture match in matches)
                    {
                        // create a leading uncolored region 
                        if (match.Index > index)
                        {
                            //Console.WriteLine("uncolored: {0} '{1}' -> {2} '{3}'", index, regionText[index], match.Index - 1, regionText[match.Index - 1]); 
                            Region uncoloredRegion = new Region(regionText.Substring(index, match.Index - index));
                            referenceNode = regions.AddAfter(referenceNode, uncoloredRegion);
                        }

                        // create a colored region
                        // Console.WriteLine("name = {0}", rule.ClassName);
                        //Console.WriteLine("colored: {0} '{1}' -> {2} '{3}'", match.Index, regionText[match.Index], match.Index + match.Length - 1, regionText[match.Index + match.Length - 1]);
                        Region coloredRegion = new Region(rule.ClassName, regionText.Substring(match.Index, match.Length));
                        referenceNode = regions.AddAfter(referenceNode, coloredRegion);

                        index = match.Index + match.Length;
                    }

                    // create a trailing uncolored region
                    if (index < regionText.Length)
                    {
                        Region uncoloredRegion = new Region(regionText.Substring(index));
                        referenceNode = regions.AddAfter(referenceNode, uncoloredRegion);
                    }

                    // remove the original node
                    regions.Remove(node);
                    node = referenceNode.Next;
                }
            }
            return (regions);
        }

        private static void writeColorizedSnippet(ICollection<Region> regions, XmlWriter writer)
        {
            foreach (Region region in regions)
            {
                // Console.WriteLine("writing {0}", region.ClassName);
                if (region.ClassName == null)
                {
                    writer.WriteString(region.Text);
                }
                else
                {
                    writer.WriteStartElement("span");
                    writer.WriteAttributeString("class", region.ClassName);
                    writer.WriteString(region.Text);
                    writer.WriteEndElement();
                }
            }
        }
        #endregion
    }

    /// <summary>
    /// Language class.
    /// </summary>
    internal class Language
    {
        #region Private members
        /// <summary>
        /// The id of the programming language.
        /// </summary>
        private string languageId;

        /// <summary>
        /// Language file extension.
        /// </summary>
        private string extension;

        /// <summary>
        /// List of colorization rules.
        /// </summary>
        private List<ColorizationRule> colorizationRules;
        #endregion

        #region Constructor
        /// <summary>
        /// Language Constructor
        /// </summary>
        /// <param name="languageId">language id</param>
        /// <param name="extension">language file extension</param>
        /// <param name="rules">colorization rules</param>
        public Language(string languageId, string extension, List<ColorizationRule> rules)
        {
            this.languageId = languageId;
            this.extension = extension;
            this.colorizationRules = rules;
        }
        #endregion

        #region Public properties
        /// <summary>
        /// Gets the languageId.
        /// </summary>
        public string LanguageId
        {
            get
            {
                return this.languageId;
            }
        }

        /// <summary>
        /// Gets the file extension
        /// </summary>
        public string Extension
        {
            get
            {
                return this.extension;
            }
        }

        /// <summary>
        /// Gets the colorization rules
        /// </summary>
        public List<ColorizationRule> ColorizationRules
        {
            get
            {
                return this.colorizationRules;
            }
        }
        #endregion

        #region Public methods
        /// <summary>
        /// Check if the language is defined.
        /// </summary>
        /// <param name="languageId">language id</param>
        /// <param name="extension">file extension</param>
        /// <returns>boolean indicating if a language is defined</returns>
        public bool IsMatch(string languageId, string extension)
        {
            if (this.languageId == languageId)
            {
                if (this.extension == extension)
                {
                    return true;
                }
                else if (this.extension == "*")
                {
                    return true;
                }
            }
            else if (this.languageId == "*")
            {
                if (this.extension == extension)
                {
                    return true;
                }

                if (this.extension == "*")
                {
                    return true;
                }
            }

            return false;
        }
        #endregion
    }

    /// <summary>
    /// Snippet class.
    /// </summary>
    internal class Snippet
    {
        #region Private Members
        /// <summary>
        /// snippet content.
        /// </summary>
        private string content;

        /// <summary>
        /// snippet language
        /// </summary>
        private Language language;
        #endregion

        #region Constructor
        /// <summary>
        /// Constructor for Snippet class.
        /// </summary>
        /// <param name="content">snippet content</param>
        /// <param name="language">snippet language</param>
        public Snippet(string content, Language language)
        {
            this.content = content;
            this.language = language;
        }
        #endregion

        #region Public properties
        /// <summary>
        /// Gets the snippet content.
        /// </summary>
        public string Content
        {
            get
            {
                return this.content;
            }
        }

        /// <summary>
        /// Gets the snippet language.
        /// </summary>
        public Language Language
        {
            get 
            {
                return this.language;
            }
        }
        #endregion
    }

    internal class SnippetCache
    {
        private int _cacheSize = 100;

        private LinkedList<String> lruLinkedList;

        private Dictionary<string, IndexedExample> cache;

        private Dictionary<string, string> _approvalIndex;
        private Dictionary<string, string> _languageMap;
        private List<Language> _languages;
        private Dictionary<string, Object> _excludedUnits;

        public SnippetCache(int cacheSize, Dictionary<string, string> approvalIndex, Dictionary<string, string> languageMap, List<Language> languages, Dictionary<string, Object> excludedUnits)
		{
			_cacheSize = cacheSize;
            _approvalIndex = approvalIndex;
            _languageMap = languageMap;
            _languages = languages;
            _excludedUnits = excludedUnits;

            cache = new Dictionary<string, IndexedExample>(_cacheSize);

			lruLinkedList = new LinkedList<string>();
		}

        public List<Snippet> GetContent(string examplePath, SnippetIdentifier snippetId)
        {
            
            // get the example containing the identifier
            IndexedExample exampleIndex = GetCachedExample(examplePath);
            if (exampleIndex == null) 
                return (null);

            // 
            return exampleIndex.GetContent(snippetId);
        }
               
        private IndexedExample GetCachedExample(string examplePath)
        {
            IndexedExample exampleIndex;
            if (cache.TryGetValue(examplePath, out exampleIndex))
            {
                // move the file from its current position to the head of the lru linked list
                lruLinkedList.Remove(exampleIndex.ListNode);
                lruLinkedList.AddFirst(exampleIndex.ListNode);
            }
            else
            {
                // not in the cache, so load and index a new example
                exampleIndex = new IndexedExample(examplePath, _approvalIndex, _languageMap, _languages, _excludedUnits);
                if (cache.Count >= _cacheSize)
                {
                    // the cache is full
                    // the last node in the linked list has the path of the next file to remove from the cache
                    if (lruLinkedList.Last != null)
                    {
                        cache.Remove(lruLinkedList.Last.Value);
                        lruLinkedList.RemoveLast();
                    }
                }
                // add the new file to the cache and to the head of the lru linked list
                cache.Add(examplePath, exampleIndex);
                exampleIndex.ListNode = lruLinkedList.AddFirst(examplePath);
            }
            return (exampleIndex);
        }


    }

    internal class IndexedExample
    {
        /// <summary>
        /// snippet store.
        /// </summary>
        private Dictionary<SnippetIdentifier, List<Snippet>> exampleSnippets = new Dictionary<SnippetIdentifier, List<Snippet>>();
        private Dictionary<string, string> _approvalIndex;
        private Dictionary<string, string> _languageMap;
        private List<Language> _languages;
        private Dictionary<string, Object> _excludedUnits;

        public IndexedExample(string examplePath, Dictionary<string, string> approvalIndex, Dictionary<string, string> languageMap, List<Language> languages, Dictionary<string, Object> excludedUnits)
        {
            _approvalIndex = approvalIndex;
            _languageMap = languageMap;
            _languages = languages;
            _excludedUnits = excludedUnits;

            // load all the snippets under the specified example path
            this.ParseExample(new DirectoryInfo(examplePath));
        }

        public List<Snippet> GetContent(SnippetIdentifier identifier)
        {
            if (exampleSnippets.ContainsKey(identifier))
                return exampleSnippets[identifier];
            else
                return null;
        }
                
        private LinkedListNode<string> listNode;
        public LinkedListNode<string> ListNode
        {
            get
            {
                return (listNode);
            }
            set
            {
                listNode = value;
            }
        }

        /// <summary>
        /// Check whether the snippet unit is approved
        /// </summary>
        /// <param name="unit">unit directory</param>
        /// <returns>boolean indicating whether the snippet unit is approved</returns>
        private bool isApprovedUnit(DirectoryInfo unit)
        {
            string sampleName = unit.Parent.Name.ToLower(CultureInfo.InvariantCulture);
            string unitName = unit.Name.ToLower(CultureInfo.InvariantCulture);

            // return false if the unit name is in the list of names to exclude
            if (_excludedUnits.ContainsKey(unitName))
                return false;

            // if no approval log is specified, all snippets are approved by default
            if (_approvalIndex.Count == 0)
                return true;

            if (_approvalIndex.ContainsKey(Path.Combine(sampleName, unitName)))
            {
                return true;
            }

            return false;
        }

        /// <summary>
        /// Parse the example directory.
        /// </summary>
        /// <param name="unit">unit directory</param>
        private void ParseExample(DirectoryInfo exampleDirectory)
        {
            // process the approved language-specific unit directories for this example
            DirectoryInfo[] unitDirectories = exampleDirectory.GetDirectories();

            foreach (DirectoryInfo unit in unitDirectories)
            {
                if (this.isApprovedUnit(unit))
                    this.ParseUnit(unit);
            }
        }

        /// <summary>
        /// Parse the unit directory for language files.
        /// </summary>
        /// <param name="unit">unit directory containing a language-specific version of the example</param>
        private void ParseUnit(DirectoryInfo unit)
        {
            // the language is the Unit Directory name, or the language id mapped to that name
            string language = unit.Name;
            if (_languageMap.ContainsKey(language.ToLower()))
                language = _languageMap[language.ToLower()];

            ParseDirectory(unit, language, unit.Parent.Name);
        }

        /// <summary>
        /// Parse an example subdir looking for source files containing snipppets.
        /// </summary>
        /// <param name="directory">The directory to parse</param>
        /// <param name="language">the id of a programming language</param>
        /// <param name="exampleName">the name of the example</param>
        private void ParseDirectory(DirectoryInfo directory, string language, string exampleName)
        {
            // parse the files in this directory
            FileInfo[] files = directory.GetFiles();
            foreach (FileInfo file in files)
                ParseFile(file, language, exampleName);

            // recurse to get files in any subdirectories
            DirectoryInfo[] subdirectories = directory.GetDirectories();
            foreach (DirectoryInfo subdirectory in subdirectories)
                ParseDirectory(subdirectory, language, exampleName);
        }

        /// <summary>
        /// Parse the language files to retrieve the snippet content.
        /// </summary>
        /// <param name="file">snippet file</param>
        /// <param name="language">snippet language</param>
        /// <param name="example">name of the example that contains this file</param>
        private void ParseFile(FileInfo file, string language, string exampleName)
        {
            string snippetLanguage = string.Empty;

            // The snippet language is the name (or id mapping) of the Unit folder
            // unless the file extension is .xaml
            // NOTE: this is just preserving the way ExampleBuilder handled it (which we can change when we're confident there are no unwanted side-effects)
            if (file.Extension.ToLower() == ".xaml")
                snippetLanguage = "XAML";
            else
                snippetLanguage = language;

            // get the text in the file
            StreamReader reader = file.OpenText();
            string text = reader.ReadToEnd();
            reader.Close();

            this.parseSnippetContent(text, snippetLanguage, file.Extension, exampleName);
        }

        /// <summary>
        /// Parse the snippet content.
        /// </summary>
        /// <param name="text">content to be parsed</param>
        /// <param name="language">snippet language</param>
        /// <param name="extension">file extension</param>
        /// <param name="example">snippet example</param>
        private void parseSnippetContent(string text, string language, string extension, string example)
        {
            // parse the text for snippets
            for (Match match = find.Match(text); match.Success; match = find.Match(text, match.Index + 10))
            {
                string snippetIdentifier = match.Groups["id"].Value;
                string snippetContent = match.Groups["tx"].Value;
                snippetContent = clean.Replace(snippetContent, "\n");

                //if necessary, clean one more time to catch snippet comments on consecutive lines
                if (clean.Match(snippetContent).Success)
                {
                    snippetContent = clean.Replace(snippetContent, "\n");
                }

                snippetContent = cleanAtStart.Replace(snippetContent, "");
                snippetContent = cleanAtEnd.Replace(snippetContent, "");

                // get the language/extension from our languages List, which may contain colorization rules for the language
                Language snippetLanguage = new Language(language, extension, null);
                foreach (Language lang in _languages)
                {
                    if (!lang.IsMatch(language, extension))
                        continue;
                    snippetLanguage = lang;
                    break;
                }

                SnippetIdentifier identifier = new SnippetIdentifier(example, snippetIdentifier);
                // BUGBUG: i don't think this ever happens, but if it did we should write an error
                if (!IsLegalXmlText(snippetContent))
                {
                    // WriteMessage(MessageLevel.Warn, String.Format("Snippet '{0}' language '{1}' contains illegal characters.", identifier.ToString(), snippetLanguage.LanguageId));
                    continue;
                }

                snippetContent = StripLeadingSpaces(snippetContent);

                // Add the snippet information to dictionary
                Snippet snippet = new Snippet(snippetContent, snippetLanguage);
                List<Snippet> values;

                if (!this.exampleSnippets.TryGetValue(identifier, out values))
                {
                    values = new List<Snippet>();
                    this.exampleSnippets.Add(identifier, values);
                }
                values.Add(snippet);
            }
        }

        private bool IsLegalXmlText(string text)
        {
            foreach (char c in text)
            {
                if (!IsLegalXmlCharacter(c)) return (false);
            }
            return (true);
        }

        private bool IsLegalXmlCharacter(char c)
        {
            if (c < 0x20)
            {
                return ((c == 0x09) || (c == 0x0A) || (c == 0x0D));
            }
            else
            {
                if (c < 0xD800)
                {
                    return (true);
                }
                else
                {
                    return ((c >= 0xE000) && (c <= 0xFFFD));
                }
            }
        }

        private static string StripLeadingSpaces(string text)
        {

            if (text == null) throw new ArgumentNullException("text");

            // split the text into lines
            string[] stringSeparators = new string[] { "\r\n" };
            string[] lines = text.Split(stringSeparators, StringSplitOptions.None);

            // no need to do this if there is only one line
            if (lines.Length == 1) return (lines[0]);

            // figure out how many leading spaces to delete
            int spaces = Int32.MaxValue;
            for (int i = 0; i < lines.Length; i++)
            {

                string line = lines[i];

                // skip empty lines
                if (line.Length == 0) continue;

                // determine the number of leading spaces
                int index = 0;
                while (index < line.Length)
                {
                    if (line[index] != ' ') break;
                    index++;
                }

                if (index == line.Length)
                {
                    // lines that are all spaces should just be treated as empty
                    lines[i] = String.Empty;
                }
                else
                {
                    // otherwise, keep track of the minimum number of leading spaces				
                    if (index < spaces) spaces = index;
                }

            }

            // re-form the string with leading spaces deleted
            StringBuilder result = new StringBuilder();
            foreach (string line in lines)
            {
                if (line.Length == 0)
                {
                    result.AppendLine();
                }
                else
                {
                    result.AppendLine(line.Substring(spaces));
                }
            }
            // Console.WriteLine("AFTER:");
            // Console.WriteLine(result.ToString());
            return (result.ToString());

        }		

        /// <summary>
        /// Regex to find the snippet content.
        /// </summary>
        private static Regex find = new Regex(
            @"<snippet(?<id>\w+)>.*\n(?<tx>(.|\n)*?)\n.*</snippet(\k<id>)>",
            RegexOptions.IgnoreCase | RegexOptions.Compiled);

        /// <summary>
        /// Regex to clean the snippet content.
        /// </summary>
        private static Regex clean = new Regex(
            @"\n[^\n]*?<(/?)snippet(\w+)>[^\n]*?\n",
            RegexOptions.IgnoreCase | RegexOptions.Compiled);

        /// <summary>
        /// Regex to clean the start of the snippet.
        /// </summary>
        private static Regex cleanAtStart = new Regex(
            @"^[^\n]*?<(/?)snippet(\w+)>[^\n]*?\n",
            RegexOptions.IgnoreCase | RegexOptions.Compiled);

        /// <summary>
        /// Regex to clean the end of the snippet.
        /// </summary>
        private static Regex cleanAtEnd = new Regex(
            @"\n[^\n]*?<(/?)snippet(\w+)>[^\n]*?$",
            RegexOptions.IgnoreCase | RegexOptions.Compiled);

    }


}