path: root/tools/Sandcastle/Source/BuildAssembler/BuildComponents/CopyFromIndexComponent.cs

// Copyright (c) Microsoft Corporation.  All rights reserved.
//
using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Xml;
using System.Xml.XPath;
using System.Reflection;
using System.Xml.Xsl; /* for custom context stuff */

namespace Microsoft.Ddue.Tools {

    public class CopyFromIndexComponent : BuildComponent {

        // XPath search patterns

        // List of copy components
        private List<CopyComponent> components = new List<CopyComponent>();

        // what to copy
        private List<CopyCommand> copy_commands = new List<CopyCommand>();

        // a context in which to evaluate XPath expressions
        private CustomContext context = new CustomContext();

        public CopyFromIndexComponent(BuildAssembler assembler, XPathNavigator configuration)
            : base(assembler, configuration) {

            // set up the context
            XPathNodeIterator context_nodes = configuration.Select("context");
            foreach (XPathNavigator context_node in context_nodes) {
                string prefix = context_node.GetAttribute("prefix", String.Empty);
                string name = context_node.GetAttribute("name", String.Empty);
                context.AddNamespace(prefix, name);
            }

            // set up the indices
            XPathNodeIterator index_nodes = configuration.Select("index");
            foreach (XPathNavigator index_node in index_nodes) {

                // get the name of the index
                string name = index_node.GetAttribute("name", String.Empty);
                if (String.IsNullOrEmpty(name)) throw new ConfigurationErrorsException("Each index must have a unique name.");

                // get the xpath for value nodes
                string value_xpath = index_node.GetAttribute("value", String.Empty);
                if (String.IsNullOrEmpty(value_xpath)) WriteMessage(MessageLevel.Error, "Each index element must have a value attribute containing an XPath that describes index entries.");

                // get the xpath for keys (relative to value nodes)
                string key_xpath = index_node.GetAttribute("key", String.Empty);
                if (String.IsNullOrEmpty(key_xpath)) WriteMessage(MessageLevel.Error, "Each index element must have a key attribute containing an XPath (relative to the value XPath) that evaluates to the entry key.");

                // get the cache size
                int cache = 10;
                string cache_value = index_node.GetAttribute("cache", String.Empty);
                if (!String.IsNullOrEmpty(cache_value)) cache = Convert.ToInt32(cache_value);

                // create the index
                IndexedDocumentCache index = new IndexedDocumentCache(this, key_xpath, value_xpath, context, cache);

                // search the data directories for entries
                XPathNodeIterator data_nodes = index_node.Select("data");
                foreach (XPathNavigator data_node in data_nodes) {

                    string base_value = data_node.GetAttribute("base", String.Empty);
                    if (!String.IsNullOrEmpty(base_value)) base_value = Environment.ExpandEnvironmentVariables(base_value);

                    bool recurse = false;
                    string recurse_value = data_node.GetAttribute("recurse", String.Empty);
                    if (!String.IsNullOrEmpty(recurse_value)) recurse = (bool)Convert.ToBoolean(recurse_value);

                    // get the files				
                    string files = data_node.GetAttribute("files", String.Empty);
                    if (String.IsNullOrEmpty(files)) WriteMessage(MessageLevel.Error, "Each data element must have a files attribute specifying which files to index.");
                    // if ((files == null) || (files.Length == 0)) throw new ConfigurationErrorsException("When instantiating a CopyFromDirectory component, you must specify a directory path using the files attribute.");
                    files = Environment.ExpandEnvironmentVariables(files);

                    WriteMessage(MessageLevel.Info, String.Format("Searching for files that match '{0}'.", files));
                    index.AddDocuments(base_value, files, recurse);

                }
                WriteMessage(MessageLevel.Info, String.Format("Indexed {0} elements in {1} files.", index.Count, index.DocumentCount));

                Data.Add(name, index);

            }

            // get the copy commands
            XPathNodeIterator copy_nodes = configuration.Select("copy");
            foreach (XPathNavigator copy_node in copy_nodes) {

                string source_name = copy_node.GetAttribute("name", String.Empty);
                if (String.IsNullOrEmpty(source_name)) throw new ConfigurationErrorsException("Each copy command must specify an index to copy from.");

                string key_xpath = copy_node.GetAttribute("key", String.Empty);

                string source_xpath = copy_node.GetAttribute("source", String.Empty);
                if (String.IsNullOrEmpty(source_xpath)) throw new ConfigurationErrorsException("When instantiating a CopyFromDirectory component, you must specify a source xpath format using the source attribute.");

                string target_xpath = copy_node.GetAttribute("target", String.Empty);
                if (String.IsNullOrEmpty(target_xpath)) throw new ConfigurationErrorsException("When instantiating a CopyFromDirectory component, you must specify a target xpath format using the target attribute.");

                string attribute_value = copy_node.GetAttribute("attribute", String.Empty);

                string ignoreCase_value = copy_node.GetAttribute("ignoreCase", String.Empty);

                string missingEntryValue = copy_node.GetAttribute("missing-entry", String.Empty);
                string missingSourceValue = copy_node.GetAttribute("missing-source", String.Empty);
                string missingTargetValue = copy_node.GetAttribute("missing-target", String.Empty);

                IndexedDocumentCache index = (IndexedDocumentCache)Data[source_name];

                CopyCommand copyCommand = new CopyCommand(index, key_xpath, source_xpath, target_xpath, attribute_value, ignoreCase_value);
                if (!String.IsNullOrEmpty(missingEntryValue)) {
                    try {
                        copyCommand.MissingEntry = (MessageLevel)Enum.Parse(typeof(MessageLevel), missingEntryValue, true);
                    } catch (ArgumentException) {
                        WriteMessage(MessageLevel.Error, String.Format("'{0}' is not a message level.", missingEntryValue));
                    }
                }
                if (!String.IsNullOrEmpty(missingSourceValue)) {
                    try {
                        copyCommand.MissingSource = (MessageLevel)Enum.Parse(typeof(MessageLevel), missingSourceValue, true);
                    } catch (ArgumentException) {
                        WriteMessage(MessageLevel.Error, String.Format("'{0}' is not a message level.", missingSourceValue));
                    }
                }
                if (!String.IsNullOrEmpty(missingTargetValue)) {
                    try {
                        copyCommand.MissingTarget = (MessageLevel)Enum.Parse(typeof(MessageLevel), missingTargetValue, true);
                    } catch (ArgumentException) {
                        WriteMessage(MessageLevel.Error, String.Format("'{0}' is not a message level.", missingTargetValue));
                    }
                }

                copy_commands.Add(copyCommand);

            }

            XPathNodeIterator component_nodes = configuration.Select("components/component");
            foreach (XPathNavigator component_node in component_nodes) {

                // get the data to load the component
                string assembly_path = component_node.GetAttribute("assembly", String.Empty);
                if (String.IsNullOrEmpty(assembly_path)) WriteMessage(MessageLevel.Error, "Each component element must have an assembly attribute.");
                string type_name = component_node.GetAttribute("type", String.Empty);
                if (String.IsNullOrEmpty(type_name)) WriteMessage(MessageLevel.Error, "Each component element must have a type attribute.");

                // expand environment variables in the path
                assembly_path = Environment.ExpandEnvironmentVariables(assembly_path);

                //Console.WriteLine("loading {0} from {1}", type_name, assembly_path);
                try 
                {
                    Assembly assembly = Assembly.LoadFrom(assembly_path);
                    CopyComponent component = (CopyComponent)assembly.CreateInstance(type_name, false, BindingFlags.Public | BindingFlags.Instance, null, new Object[2] { component_node.Clone(), Data }, null, null);

                    if (component == null)
                    {
                        WriteMessage(MessageLevel.Error, String.Format("The type '{0}' does not exist in the assembly '{1}'.", type_name, assembly_path));
                    }
                    else
                    {
                        components.Add(component);
                    }

                }
                catch (IOException e)
                {
                    WriteMessage(MessageLevel.Error, String.Format("A file access error occured while attempting to load the build component '{0}'. The error message is: {1}", assembly_path, e.Message));
                }
                catch (BadImageFormatException e)
                {
                    WriteMessage(MessageLevel.Error, String.Format("A syntax generator assembly '{0}' is invalid. The error message is: {1}.", assembly_path, e.Message));
                }
                catch (TypeLoadException e)
                {
                    WriteMessage(MessageLevel.Error, String.Format("The type '{0}' does not exist in the assembly '{1}'. The error message is: {2}", type_name, assembly_path, e.Message));
                }
                catch (MissingMethodException e)
                {
                    WriteMessage(MessageLevel.Error, String.Format("The type '{0}' in the assembly '{1}' does not have an appropriate constructor. The error message is: {2}", type_name, assembly_path, e.Message));
                }
                catch (TargetInvocationException e)
                {
                    WriteMessage(MessageLevel.Error, String.Format("An error occured while attempting to instantiate the type '{0}' in the assembly '{1}'. The error message is: {2}", type_name, assembly_path, e.InnerException.Message));
                }
                catch (InvalidCastException e)
                {
                    WriteMessage(MessageLevel.Error, String.Format("The type '{0}' in the assembly '{1}' is not a SyntaxGenerator.", type_name, assembly_path));
                }
            }

            WriteMessage(MessageLevel.Info, String.Format("Loaded {0} copy components.", components.Count));

        }

        // the actual work of the component

        public override void Apply(XmlDocument document, string key) {

            // set the key in the XPath context
            context["key"] = key;

            // perform each copy action
            foreach (CopyCommand copy_command in copy_commands) {

                // get the source comment
                XPathExpression key_expression = copy_command.Key.Clone();
                key_expression.SetContext(context);
                // Console.WriteLine(key_expression.Expression);
                string key_value = (string)document.CreateNavigator().Evaluate(key_expression);
                // Console.WriteLine("got key '{0}'", key_value);
                XPathNavigator data = copy_command.Index.GetContent(key_value);

                if (data == null && copy_command.IgnoreCase == "true") data = copy_command.Index.GetContent(key_value.ToLower());

                // notify if no entry
                if (data == null) {
                    WriteMessage(copy_command.MissingEntry, String.Format("No index entry found for key '{0}'.", key_value));
                    continue;
                }

                // get the target node
                String target_xpath = copy_command.Target.Clone().ToString();
                XPathExpression target_expression = XPathExpression.Compile(string.Format(target_xpath, key_value));
                target_expression.SetContext(context);

                XPathNavigator target = document.CreateNavigator().SelectSingleNode(target_expression);

                // notify if no target found
                if (target == null) {
                    WriteMessage(copy_command.MissingTarget, String.Format("Target node '{0}' not found.", target_expression.Expression));
                    continue;
                }

                // get the source nodes
                XPathExpression source_expression = copy_command.Source.Clone();
                source_expression.SetContext(context);
                XPathNodeIterator sources = data.CreateNavigator().Select(source_expression);

                // append the source nodes to the target node
                int source_count = 0;
                foreach (XPathNavigator source in sources) {
                    source_count++;

                    // If attribute=true, add the source attributes to current target. 
                    // Otherwise append source as a child element to target
                    if (copy_command.Attribute == "true" && source.HasAttributes) {
                        string source_name = source.LocalName;
                        XmlWriter attributes = target.CreateAttributes();

                        source.MoveToFirstAttribute();
                        string attrFirst = target.GetAttribute(string.Format("{0}_{1}", source_name, source.Name), string.Empty);
                        if (string.IsNullOrEmpty(attrFirst)) attributes.WriteAttributeString(string.Format("{0}_{1}", source_name, source.Name), source.Value);

                        while (source.MoveToNextAttribute()) {
                            string attrNext = target.GetAttribute(string.Format("{0}_{1}", source_name, source.Name), string.Empty);
                            if (string.IsNullOrEmpty(attrNext)) attributes.WriteAttributeString(string.Format("{0}_{1}", source_name, source.Name), source.Value);
                        }
                        attributes.Close();
                    }
                    else target.AppendChild(source);
                }

                // notify if no source found
                if (source_count == 0) {
                    WriteMessage(copy_command.MissingSource, String.Format("Source node '{0}' not found.", source_expression.Expression));
                }

                foreach (CopyComponent component in components)
                {
                    component.Apply(document, key);
                }
            }
        }

        internal void WriteHelperMessage(MessageLevel level, string message) {
            WriteMessage(level, message);
        }

    }

    // the storage system

    public class IndexedDocumentCache {

        public IndexedDocumentCache(CopyFromIndexComponent component, string keyXPath, string valueXPath, XmlNamespaceManager context, int cacheSize) {

            if (component == null) throw new ArgumentNullException("component");
            if (cacheSize < 0) throw new ArgumentOutOfRangeException("cacheSize");

            this.component = component;

            try {
                keyExpression = XPathExpression.Compile(keyXPath);
            } catch (XPathException) {
                component.WriteHelperMessage(MessageLevel.Error, String.Format("The key expression '{0}' is not a valid XPath expression.", keyXPath));
            }
            keyExpression.SetContext(context);

            try {
                valueExpression = XPathExpression.Compile(valueXPath);
            } catch (XPathException) {
                component.WriteHelperMessage(MessageLevel.Error, String.Format("The value expression '{0}' is not a valid XPath expression.", valueXPath));
            }
            valueExpression.SetContext(context);

            this.cacheSize = cacheSize;

            // set up the cache
            cache = new Dictionary<string, IndexedDocument>(cacheSize);
            queue = new Queue<string>(cacheSize);
        }

        // index component to which the cache belongs
        private CopyFromIndexComponent component;

        public CopyFromIndexComponent Component {
            get {
                return (component);
            }
        }

        // search pattern for index values
        private XPathExpression valueExpression;

        public XPathExpression ValueExpression {
            get {
                return (valueExpression);
            }
        }

        // search pattern for the index keys (relative to the index value node)
        private XPathExpression keyExpression;

        public XPathExpression KeyExpression {
            get {
                return (keyExpression);
            }
        }

        // a index mapping keys to the files that contain them
        private Dictionary<string, string> index = new Dictionary<string, string>();

        public void AddDocument(string file) {

            // load the document
            IndexedDocument document = new IndexedDocument(this, file);

            // record the keys
            string[] keys = document.GetKeys();
            foreach (string key in keys) {
                if (index.ContainsKey(key)) {
                    component.WriteHelperMessage(MessageLevel.Warn, String.Format("Entries for the key '{0}' occur in both '{1}' and '{2}'. The last entry will be used.", key, index[key], file));
                }
                index[key] = file;

            }

        }

        public void AddDocuments(string wildcardPath) {
            string directory_part = Path.GetDirectoryName(wildcardPath);
            if (String.IsNullOrEmpty(directory_part)) directory_part = Environment.CurrentDirectory;
            directory_part = Path.GetFullPath(directory_part);
            string file_part = Path.GetFileName(wildcardPath);
            //Console.WriteLine("{0}::{1}", directory_part, file_part);
            string[] files = Directory.GetFiles(directory_part, file_part);
            foreach (string file in files) {
                AddDocument(file);
            }

            //Console.WriteLine(files.Length);
            documentCount += files.Length;
        }

        public void AddDocuments(string baseDirectory, string wildcardPath, bool recurse) {

            string path;
            if (String.IsNullOrEmpty(baseDirectory)) {
                path = wildcardPath;
            } else {
                path = Path.Combine(baseDirectory, wildcardPath);
            }

            AddDocuments(path);

            if (recurse) {
                string[] subDirectories = Directory.GetDirectories(baseDirectory);
                foreach (string subDirectory in subDirectories) AddDocuments(subDirectory, wildcardPath, recurse);
            }
        }

        private int documentCount;

        public int DocumentCount {
            get {
                return (documentCount);
            }
        }

        // a simple caching mechanism

        int cacheSize;

        // an improved cache

        // this cache keeps track of the order that files are loaded in, and always unloads the oldest one
        // this is better, but a document that is often accessed gets no "points", so it will eventualy be
        // thrown out even if it is used regularly

        private Dictionary<string, IndexedDocument> cache;

        private Queue<string> queue;

        public IndexedDocument GetDocument(string key) {

            // look up the file corresponding to the key
            string file;
            if (index.TryGetValue(key, out file)) {

                // now look for that file in the cache
                IndexedDocument document;
                if (!cache.TryGetValue(file, out document)) {

                    // not in the cache, so load it
                    document = new IndexedDocument(this, file);

                    // if the cache is full, remove a document
                    if (cache.Count >= cacheSize) {
                        string fileToUnload = queue.Dequeue();
                        cache.Remove(fileToUnload);
                    }

                    // add it to the cache
                    cache.Add(file, document);
                    queue.Enqueue(file);

                }

                // XPathNavigator content = document.GetContent(key);
                return (document);

            } else {
                // there is no such key
                return (null);
            }

        }


        public XPathNavigator GetContent(string key) {

            IndexedDocument document = GetDocument(key);
            if (document == null) {
                return (null);
            } else {
                return (document.GetContent(key));
            }

        }

        public int Count {
            get {
                return (index.Count);
            }
        }

    }

    // a file that we have indexed

    public class IndexedDocument {

        public IndexedDocument(IndexedDocumentCache cache, string file) {

            if (cache == null) throw new ArgumentNullException("cache");
            if (file == null) throw new ArgumentNullException("file");

            // remember the file
            this.file = file;

            // load the document
            try {
                XPathDocument document = new XPathDocument(file, XmlSpace.Preserve);
                //XPathDocument document = new XPathDocument(file);

                // search for value nodes
                XPathNodeIterator valueNodes = document.CreateNavigator().Select(cache.ValueExpression);
                // Console.WriteLine("found {0} instances of '{1}' (key xpath is '{2}')", valueNodes.Count, valueExpression.Expression, keyExpression.Expression);

                // get the key string for each value node and record it in the index
                foreach (XPathNavigator valueNode in valueNodes) {

                    XPathNavigator keyNode = valueNode.SelectSingleNode(cache.KeyExpression);
                    if (keyNode == null) {
                        // Console.WriteLine("null key");
                        continue;
                    }

                    string key = keyNode.Value;
                    index[key] = valueNode;
                    if (!index.ContainsKey(key)) {
                        //index.Add(key, valueNode);
                    } else {
                        // Console.WriteLine("Repeat key '{0}'", key);
                    }
                }

            } catch (IOException e) {
                cache.Component.WriteHelperMessage(MessageLevel.Error, String.Format("An access error occured while attempting to load the file '{0}'. The error message is: {1}", file, e.Message));
            } catch (XmlException e) {
                cache.Component.WriteHelperMessage(MessageLevel.Error, String.Format("The indexed document '{0}' is not a valid XML document. The error message is: {1}", file, e.Message));
            }
            // Console.WriteLine("indexed {0} keys", index.Count);


        }

        // the indexed file

        private string file;

        // the index that maps keys to positions in the file		

        Dictionary<string, XPathNavigator> index = new Dictionary<string, XPathNavigator>();

        // public methods

        public string File {
            get {
                return (file);
            }
        }

        public XPathNavigator GetContent(string key) {
            XPathNavigator value = index[key];
            if (value == null) {
                return (null);
            } else {
                return (value.Clone());
            }
        }

        public string[] GetKeys() {
            string[] keys = new string[Count];
            index.Keys.CopyTo(keys, 0);
            return (keys);
        }

        public int Count {
            get {
                return (index.Count);
            }
        }

    }

    internal class CopyCommand {

        public CopyCommand(IndexedDocumentCache source_index, string key_xpath, string source_xpath, string target_xpath, string attribute_value, string ignoreCase_value) {
            this.cache = source_index;
            if (String.IsNullOrEmpty(key_xpath)) {
                // Console.WriteLine("null key xpath");
                key = XPathExpression.Compile("string($key)");
            } else {
                // Console.WriteLine("compiling key xpath '{0}'", key_xpath);
                key = XPathExpression.Compile(key_xpath);
            }
            source = XPathExpression.Compile(source_xpath);
            target = target_xpath;
            attribute = attribute_value;
            ignoreCase = ignoreCase_value;
        }

        private IndexedDocumentCache cache;

        private XPathExpression key;

        private XPathExpression source;

        private String target;

        private String attribute;

        private String ignoreCase;

        private MessageLevel missingEntry = MessageLevel.Ignore;

        private MessageLevel missingSource = MessageLevel.Ignore;

        private MessageLevel missingTarget = MessageLevel.Ignore;

        public IndexedDocumentCache Index {
            get {
                return (cache);
            }
        }

        public XPathExpression Key {
            get {
                return (key);
            }
        }

        public XPathExpression Source {
            get {
                return (source);
            }
        }

        public String Target {
            get {
                return (target);
            }
        }

        public String Attribute 
        {
            get 
            {
                return (attribute);
            }
        }

        public String IgnoreCase 
        {
            get 
            {
                return (ignoreCase);
            }
        }

        public MessageLevel MissingEntry {
            get {
                return (missingEntry);
            } 
            set {
                missingEntry = value;
            }
        }

        public MessageLevel MissingSource {
            get {
                return (missingSource);
            } 
            set {
                missingSource = value;
            }
        }

        public MessageLevel MissingTarget {
            get {
                return (missingTarget);
            } 
            set {
                missingTarget = value;
            }
        }

    }
    
    // the abstract CopyComponent
    public abstract class CopyComponent
    {

        public CopyComponent(XPathNavigator configuration, Dictionary<string, object> data) { }

        public abstract void Apply(XmlDocument document, string key);

    }
}