[Library Ordering] Automatically sort libraries in their dependency order, then apply user order to resolve any conflicts at the same dependency level.

Author: kudaba

Sharpmake.UnitTests/DependencyPropagationTest.cs

@@ -1086,17 +1086,58 @@ public void LibInheritLibAndDLLPrivate()
                 Assert.True(conf.DependenciesLibraryFiles.ContainsElement("LibDependOnDLLProject"));
             }
         }
+
+        [Test]
+        public void AutoDependencyOrder()
+        {
+            var project = GetProject<SharpmakeProjects.ProjectDependencyOrder>();
+            Assert.IsNotNull(project);
+            Assert.IsNotNull(project);
+
+            foreach (var conf in project.Configurations)
+            {
+                Assert.That(conf.ResolvedDependencies.Count, Is.EqualTo(4));
+
+                Assert.True(conf.ResolvedDependencies.ContainsProjectType(typeof(SharpmakeProjects.NoDependencyProject1)));
+                Assert.True(conf.ResolvedDependencies.ContainsProjectType(typeof(SharpmakeProjects.NoDependencyProject2)));
+                Assert.True(conf.ResolvedDependencies.ContainsProjectType(typeof(SharpmakeProjects.OnePrivateDependencyProject)));
+                Assert.True(conf.ResolvedDependencies.ContainsProjectType(typeof(SharpmakeProjects.TwoPrivateDependenciesProject)));
+
+                int rootProjectOrder = conf.TargetFileOrderNumber;
+                int project1Order = conf.ResolvedDependencies.OfProjectType(typeof(SharpmakeProjects.NoDependencyProject1)).First().TargetFileOrderNumber;
+                int project2Order = conf.ResolvedDependencies.OfProjectType(typeof(SharpmakeProjects.NoDependencyProject2)).First().TargetFileOrderNumber;
+                int onePrivateOrder = conf.ResolvedDependencies.OfProjectType(typeof(SharpmakeProjects.OnePrivateDependencyProject)).First().TargetFileOrderNumber;
+                int twoPrivateOrder = conf.ResolvedDependencies.OfProjectType(typeof(SharpmakeProjects.TwoPrivateDependenciesProject)).First().TargetFileOrderNumber;
+
+                // I can't know the exact values but dependencies should at least appear as higher values
+                Assert.Less(rootProjectOrder, project1Order);
+                Assert.Less(rootProjectOrder, project2Order);
+
+                Assert.Less(onePrivateOrder, project1Order);
+                Assert.Less(onePrivateOrder, project2Order);
+
+                Assert.Less(twoPrivateOrder, project1Order);
+                Assert.Less(twoPrivateOrder, project2Order);
+
+                Assert.AreEqual(onePrivateOrder, twoPrivateOrder);
+                Assert.AreEqual(project1Order, project2Order);
+            }
+        }
     }
 
     public static class UTestUtilities
     {
         public static readonly string IncludeFolder = "include";
         public static readonly string LibOutputFolder = "lib_output";
 
-        public static bool ContainsProjectType(this System.Collections.Generic.IEnumerable<Project.Configuration> dependencies, System.Type projectType)
+        public static bool ContainsProjectType(this IEnumerable<Project.Configuration> dependencies, Type projectType)
         {
             return dependencies.Select(x => x.Project.GetType()).Contains(projectType);
         }
+        public static IEnumerable<Project.Configuration> OfProjectType(this IEnumerable<Project.Configuration> dependencies, Type projectType)
+        {
+            return dependencies.Where(x => x.Project.GetType() == projectType);
+        }
 
         public static bool ContainsProjectType(this IEnumerable<DotNetDependency> dependencies, Type projectType)
         {
@@ -1741,5 +1782,20 @@ public void ConfigureAll(Configuration conf, Target target)
                 conf.AddPrivateDependency<OnePrivateDependencyProject>(target);
             }
         }
+
+        [Sharpmake.Generate]
+        public class ProjectDependencyOrder : UTestUtilities.UnitTestCommonProject
+        {
+            public ProjectDependencyOrder() { }
+
+            [Configure()]
+            public void ConfigureAll(Configuration conf, Target target)
+            {
+                conf.AddPrivateDependency<OnePrivateDependencyProject>(target);
+                conf.AddPrivateDependency<TwoPrivateDependenciesProject>(target);
+                conf.AddPrivateDependency<NoDependencyProject1>(target);
+                conf.AddPrivateDependency<NoDependencyProject2>(target);
+            }
+        }
     }
 }

Sharpmake/Project.Configuration.cs

@@ -1151,7 +1151,7 @@ public void AddDependencyBuiltTargetLibraryFile(string libraryFile, int orderNum
             {
                 if (_linkState != LinkState.Linking)
                     throw new Error($"Cannot add built target lib '{libraryFile}' outside of the link process of the Project.Configuration");
-                DependenciesBuiltTargetsLibraryFiles.Add(libraryFile, orderNumber);
+                DependenciesBuiltTargetsLibraryFiles.Add(libraryFile, orderNumber, OrderableStrings.OrderResolve.Greater);
             }
 
             public OrderableStrings DependenciesForceUsingFiles = new OrderableStrings();
@@ -1697,10 +1697,33 @@ internal void Resolve(Resolver resolver)
             public string TargetFileFullNameWithExtension { get; internal set; } = "[conf.TargetFileFullName][conf.TargetFileFullExtension]";
 
             /// <summary>
-            /// Gets or sets the ordering index of the target when added as a library to another
+            /// Sets the ordering index of the target when added as a library to another. This will only affect the order of libraries
+            /// at the same dependency level. Dependent libraries always appear first.
             /// project.
+            /// Gets the combined user order and dependency order.
             /// </summary>
-            public int TargetFileOrderNumber = 0;
+            public int TargetFileOrderNumber
+            {
+                get
+                {
+                    return _targetFileOrderNumber + _autoFileOrderNumer;
+                }
+                set
+                {
+                    if (value <= -_autoFileNumberStep/2 || value >= _autoFileNumberStep/2)
+                    {
+                        throw new ArgumentOutOfRangeException(string.Format("TargetFileOrderNumber should only be between {0} and {1}. Got {2}",
+                            -_autoFileNumberStep / 2, _autoFileNumberStep / 2, value));
+                    }
+                    _targetFileOrderNumber = value;
+                }
+            }
+
+            internal int _targetFileOrderNumber = 0;
+            // Used internaly to order by dependency before applying TargetFileOrderNumber or other ordering
+            internal int _autoFileOrderNumer = 0;
+            // Make depdencies 1000 items from each other, this means that a user would need to put huge numbers in target file orders to cause cross over
+            internal const int _autoFileNumberStep = 1000;
 
             /// <summary>
             /// Gets or sets the ordering index of the library paths when added as a library to
@@ -2856,6 +2879,21 @@ internal DependencyNode(Configuration inConfiguration, DependencySetting inDepen
                     _dependencySetting = inDependencySetting;
                 }
 
+                internal void UpdateAutoDependencies()
+                {
+                    // Configurations are shared so it's likely the dependency has been processed already so don't process it again
+                    // dependency roots will always be 0, but also have no children, anyone else will have a non-zero value once set
+                    // and that value should remain stable each time the dependency tree is walked.
+                    if (_configuration._autoFileOrderNumer == 0)
+                    {
+                        foreach (var child in _childNodes)
+                        {
+                            child.Key.UpdateAutoDependencies();
+                            _configuration._autoFileOrderNumer = Math.Min(_configuration._autoFileOrderNumer, child.Key._configuration._autoFileOrderNumer - 1000);
+                        }
+                    }
+                }
+
                 internal Configuration _configuration;
                 internal DependencySetting _dependencySetting;
                 internal Dictionary<DependencyNode, DependencyType> _childNodes = new Dictionary<DependencyNode, DependencyType>();
@@ -3105,8 +3143,9 @@ internal void Link(Builder builder)
                                     if (dependencySetting.HasFlag(DependencySetting.LibraryPaths))
                                         DependenciesOtherLibraryPaths.AddRange(dependency.LibraryPaths);
 
+                                    // Use dependency.TargetFileOrderNumber to make sure to group dependent libraries by their dependencies
                                     if (dependencySetting.HasFlag(DependencySetting.LibraryFiles))
-                                        DependenciesOtherLibraryFiles.AddRange(dependency.LibraryFiles);
+                                        DependenciesOtherLibraryFiles.AddRange(dependency.LibraryFiles, dependency.TargetFileOrderNumber, OrderableStrings.OrderResolve.Greater);
 
                                     if (dependencySetting.HasFlag(DependencySetting.ForceUsingAssembly))
                                         DependenciesForceUsingFiles.AddRange(dependency.ForceUsingFiles);
@@ -3138,8 +3177,9 @@ internal void Link(Builder builder)
                                         if (dependencySetting.HasFlag(DependencySetting.LibraryPaths))
                                             DependenciesOtherLibraryPaths.AddRange(dependency.LibraryPaths);
 
+                                        // Use dependency.TargetFileOrderNumber to make sure to group dependent libraries by their dependencies
                                         if (dependencySetting.HasFlag(DependencySetting.LibraryFiles))
-                                            DependenciesOtherLibraryFiles.AddRange(dependency.LibraryFiles);
+                                            DependenciesOtherLibraryFiles.AddRange(dependency.LibraryFiles, dependency.TargetFileOrderNumber, OrderableStrings.OrderResolve.Greater);
                                     }
                                 }
 
@@ -3303,6 +3343,7 @@ static private DependencyNode BuildDependencyNodeTree(Builder builder, Configura
 
                 Stack<DependencyNode> visiting = new Stack<DependencyNode>();
                 visiting.Push(rootNode);
+
                 while (visiting.Count > 0)
                 {
                     DependencyNode visitedNode = visiting.Pop();
@@ -3340,6 +3381,7 @@ static private DependencyNode BuildDependencyNodeTree(Builder builder, Configura
                             if (!visitedConfiguration._dependenciesSetting.TryGetValue(pair, out dependencySetting))
                                 dependencySetting = DependencySetting.Default;
 
+                            // We use steps of 1000 to allow for related libraries to be grouped alongside their dependencies
                             DependencyNode childNode = new DependencyNode(dependencyConf, dependencySetting);
                             System.Diagnostics.Debug.Assert(!visitedNode._childNodes.ContainsKey(childNode));
                             visitedNode._childNodes.Add(childNode, dependencyType);
@@ -3349,6 +3391,9 @@ static private DependencyNode BuildDependencyNodeTree(Builder builder, Configura
                     }
                 }
 
+                // update dependency hierarchy
+                rootNode.UpdateAutoDependencies();
+
                 return rootNode;
             }
 

Sharpmake/Strings.cs

@@ -253,7 +253,14 @@ public void Add(string item)
                 _list.Add(new StringEntry(item));
         }
 
-        public void Add(string item, int orderNumber)
+        public enum OrderResolve
+        {
+            None,
+            Less,
+            Greater
+        }
+
+        public void Add(string item, int orderNumber, OrderResolve resolveMethod = OrderResolve.None)
         {
             if (_hashSet.Add(item))
                 _list.Add(new StringEntry(item, orderNumber));
@@ -268,9 +275,22 @@ public void Add(string item, int orderNumber)
                             _list[i] = new StringEntry(item, orderNumber);
                         else if (_list[i].OrderNumber != orderNumber)
                         {
-                            throw new Error(
-                                "Cannot specify 2 different non-zero order number for \"" +
-                                item + "\": " + _list[i].OrderNumber + " and " + orderNumber);
+                            if (resolveMethod == OrderResolve.Less)
+                            {
+                                if (orderNumber < _list[i].OrderNumber)
+                                    _list[i] = new StringEntry(item, orderNumber);
+                            }
+                            else if (resolveMethod == OrderResolve.Greater)
+                            {
+                                if (orderNumber > _list[i].OrderNumber)
+                                    _list[i] = new StringEntry(item, orderNumber);
+                            }
+                            else
+                            {
+                                throw new Error(
+                                    "Cannot specify 2 different non-zero order number for \"" +
+                                    item + "\": " + _list[i].OrderNumber + " and " + orderNumber);
+                            }
                         }
                     }
                 }
@@ -283,18 +303,20 @@ public void AddRange(IEnumerable<string> collection)
                 Add(item);
         }
 
-        public void AddRange(OrderableStrings collection)
+        public void AddRange(OrderableStrings collection, int outerOrderNumber = 0, OrderResolve resolveMethod = OrderResolve.None)
         {
             List<StringEntry> existingEntriesToAdd = null;
             foreach (var entry in collection._list)
             {
+                var newEntry = new StringEntry(entry.StringValue, entry.OrderNumber + outerOrderNumber);
+
                 if (_hashSet.Add(entry.StringValue))
-                    _list.Add(entry);
-                else if (entry.OrderNumber != 0)  // make sure to have orderNumber
+                    _list.Add(newEntry);
+                else if (newEntry.OrderNumber != 0)  // make sure to have orderNumber
                 {
                     if (existingEntriesToAdd == null)
                         existingEntriesToAdd = new List<StringEntry>();
-                    existingEntriesToAdd.Add(entry);
+                    existingEntriesToAdd.Add(newEntry);
                 }
             }
             if (existingEntriesToAdd != null)
@@ -309,9 +331,22 @@ public void AddRange(OrderableStrings collection)
                             _list[i] = new StringEntry(_list[i].StringValue, orderNumber);
                         else if (_list[i].OrderNumber != orderNumber)
                         {
-                            throw new Error(
-                                "Cannot specify 2 different non-zero order number for \"" +
-                                _list[i].StringValue + "\": " + _list[i].OrderNumber + " and " + orderNumber);
+                            if (resolveMethod == OrderResolve.Less)
+                            {
+                                if (orderNumber < _list[i].OrderNumber)
+                                    _list[i] = new StringEntry(_list[i].StringValue, orderNumber);
+                            }
+                            else if (resolveMethod == OrderResolve.Greater)
+                            {
+                                if (orderNumber > _list[i].OrderNumber)
+                                    _list[i] = new StringEntry(_list[i].StringValue, orderNumber);
+                            }
+                            else
+                            {
+                                throw new Error(
+                                    "Cannot specify 2 different non-zero order number for \"" +
+                                    _list[i].StringValue + "\": " + _list[i].OrderNumber + " and " + orderNumber);
+                            }
                         }
                     }
                 }