Include Yaml

CHANGELOG.md

@@ -34,6 +34,8 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s
 - Added an `external_surfaces` transform filter, that can be used to reduce memory requirements in pipelines where you plan to only process the external faces of a data set.
 - Added a `declare_fields` action, that allows users to explicitly list the fields to return for field filtering. This option avoids complex field parsing logic.
 - Added a 2d camera mode (`camera/2d: [left, right, bottom, top]`) to scene render cameras and the `project_2d` (scalar rendering) filter cameras.
+- Added support for `include` keyword to include children from yaml files in an input node trees
+
 
 ### Changed
 - Changed the replay utility's binary names such that `replay_ser` is now `ascent_replay` and `raplay_mpi` is now `ascent_replay_mpi`. This will help prevent potential name collisions with other tools that also have replay utilities.

src/docs/sphinx/Actions/Examples.rst

@@ -623,6 +623,21 @@ Resulting image:
 
 .. image:: examples/milk_chocolate100.png
 
+An example of passing a color table using yaml include
+-------------------------------------------
+
+YAML actions:
+
+.. literalinclude:: examples/include_color_table.yaml
+
+Included YAML with color table information:
+
+.. literalinclude:: examples/color_table.yaml
+
+Resulting image:
+
+.. image:: examples/milk_chocolate100.png
+
 An example if using the composite vector filter  to compose three scalar fields into a vector.
 -----------------------------------------------------------------------------------------------
 

src/docs/sphinx/Actions/examples/color_table.yaml

@@ -0,0 +1,7 @@
+color_table: 
+  control_points: 
+    r: [0.23, 0.48, 0.99]
+    g: [0.08, 0.23, 1.0]
+    b: [0.08, 0.04, 0.9]
+    a: [1.0, 1.0, 1.0]
+    position: [0.0, 0.5, 1.0]

src/docs/sphinx/Actions/examples/include_color_table.yaml

@@ -0,0 +1,15 @@
+- 
+  action: "add_scenes"
+  scenes: 
+    s1: 
+      plots: 
+        p1: 
+          type: "pseudocolor"
+          field: "braid"
+          include: "./src/docs/sphinx/Actions/examples/color_table.yaml"
+      image_prefix: "./_output/milk_chocolate"
+      renders: 
+        r1: 
+          image_width: 512
+          image_height: 512
+          image_prefix: "./_output/milk_chocolate"

src/libs/ascent/ascent.cpp

@@ -77,6 +77,101 @@ check_for_file(const std::string &file_name,
 
 }
 
+//-----------------------------------------------------------------------------
+void
+load_included_files_in_node_tree(conduit::Node &node, int mpi_comm_id)
+{
+    // This function recursively traverses the node tree searching for include statements
+    // If one is found, the node attempts to load the file's contents and add it to the node tree
+    int comm_size = 1;
+    int rank = 0;
+
+#ifdef ASCENT_MPI_ENABLED
+    if(mpi_comm_id == -1)
+    {
+      // do nothing, an error will be thrown later
+      // so we can respect the exception handling
+      return;
+    }
+    MPI_Comm mpi_comm = MPI_Comm_f2c(mpi_comm_id);
+    MPI_Comm_size(mpi_comm, &comm_size);
+    MPI_Comm_rank(mpi_comm, &rank);
+#endif
+
+    if(node.has_child("include"))
+    {
+        int include_file_valid = 0;
+        std::string emsg = "";
+        std::string file_name = "";
+
+        // Only want to update the node on rank 0
+        if (rank == 0) {
+            file_name = node.fetch("include").as_string();
+            node.remove_child("include");
+
+            // Determine file protocol from file extension
+            std::string curr,next;
+            std::string protocol = "json";
+            conduit::utils::rsplit_string(file_name,
+                                            ".",
+                                            curr,
+                                            next);
+            if(curr == "yaml")
+            {
+                protocol = "yaml";
+            }
+
+            // Try loading in the included path
+            try
+            {
+                conduit::Node file_node;
+                file_node.load(file_name, protocol);
+                node.update(file_node);
+                include_file_valid = 1;
+            }
+            catch(conduit::Error &e)
+            {
+                include_file_valid = 0;
+                emsg = e.message();
+            }
+        }
+
+#ifdef ASCENT_MPI_ENABLED
+        // make sure all ranks error if the parsing on rank 0 failed.
+        MPI_Bcast(&include_file_valid, 1, MPI_INT, 0, mpi_comm);
+
+        // Pass the error to all ranks so the error message matches
+        conduit::Node n_emsg;
+        if(rank == 0)
+        {
+        n_emsg.set(emsg);
+        }
+
+        conduit::relay::mpi::broadcast_using_schema(n_emsg, 0, mpi_comm);
+        emsg = n_emsg.as_string();
+#endif
+
+        if(include_file_valid == 0)
+        {
+            // Raise Error
+            ASCENT_ERROR("Failed to load actions file: " << file_name
+                        << "\n" << emsg);
+        }
+
+#ifdef ASCENT_MPI_ENABLED
+        // If successful, make sure that all ranks received the updated node
+        relay::mpi::broadcast_using_schema(node, 0, mpi_comm);
+#endif
+    }
+
+    // If there are any children, recurse over the children
+    // This includes any newly included children
+    for (index_t i = 0; i<node.number_of_children(); i++)
+    {
+        load_included_files_in_node_tree(node[i], mpi_comm_id);
+    }
+}
+
 //-----------------------------------------------------------------------------
 int
 ParRank(int comm_id)
@@ -242,11 +337,15 @@ CheckForSettingsFile(std::string file_name,
         ASCENT_ERROR("Failed to load actions file: " << file_name
                      << "\n" << emsg);
     }
+
 #ifdef ASCENT_MPI_ENABLED
     relay::mpi::broadcast_using_schema(node, 0, mpi_comm);
 #endif
+
+    detail::load_included_files_in_node_tree(node, mpi_comm_id);
 }
 
+
 //-----------------------------------------------------------------------------
 void
 Ascent::open(const conduit::Node &options)
@@ -276,6 +375,8 @@ Ascent::open(const conduit::Node &options)
         int par_rank = detail::ParRank(comm_id);
         int par_size = detail::ParSize(comm_id);
 
+        detail::load_included_files_in_node_tree(processed_opts, comm_id);
+
         CheckForSettingsFile(opts_file,
                              processed_opts,
                              true,
@@ -629,6 +730,7 @@ Ascent::execute(const conduit::Node &actions)
             int mpi_comm_id = m_options["mpi_comm"].to_int();
 
             Node processed_actions(actions);
+            detail::load_included_files_in_node_tree(processed_actions, mpi_comm_id);
 
             if(m_actions_file == "<<UNSET>>")
             {

src/tests/ascent/t_ascent_ascent_runtime.cpp

@@ -269,6 +269,7 @@ TEST(ascent_pipeline, test_register_transform)
     EXPECT_TRUE(check_test_image(output_file));
 }
 
+//-----------------------------------------------------------------------------
 TEST(ascent_pipeline, test_empty_pipeline_filter)
 {
     Ascent ascent;
@@ -311,4 +312,56 @@ TEST(ascent_pipeline, test_empty_pipeline_filter)
     }
 
     EXPECT_TRUE(error_message);
+}
+
+//-----------------------------------------------------------------------------
+TEST(ascent_pipeline, test_include_yaml)
+{
+    //
+    // Create example mesh.
+    //
+    Node data, verify_info;
+    conduit::blueprint::mesh::examples::braid("hexs",
+                                               EXAMPLE_MESH_SIDE_DIM,
+                                               EXAMPLE_MESH_SIDE_DIM,
+                                               EXAMPLE_MESH_SIDE_DIM,
+                                               data);
+
+    EXPECT_TRUE(conduit::blueprint::mesh::verify(data,verify_info));
+    string output_path = prepare_output_dir();
+    string output_file = conduit::utils::join_file_path(output_path,
+                                                        "tout_render_with_yaml_included_color_table");
+    // remove old images before rendering
+    remove_test_image(output_file);
+
+    //
+    // Create the actions.
+    //
+    conduit::Node scenes;
+    scenes["s1/plots/p1/type"] = "pseudocolor";
+    scenes["s1/plots/p1/field"] = "braid";
+    scenes["s1/plots/p1/include"] = conduit::utils::join_file_path(ASCENT_T_DATA_DIR, "color_table.yaml");
+    scenes["s1/image_prefix"] = output_file;
+
+    conduit::Node actions;
+    conduit::Node &add_plots = actions.append();
+    add_plots["action"] = "add_scenes";
+    add_plots["scenes"] = scenes;
+    actions.print();
+
+    //
+    // Run Ascent
+    //
+    Ascent ascent;
+    Node ascent_opts;
+    Node ascent_info;
+    ascent_opts["runtime/type"] = "ascent";
+    ascent.open(ascent_opts);
+    ascent.publish(data);
+    ascent.execute(actions);
+    ascent.info(ascent_info);
+    ascent.close();
+
+    // check that we created an image
+    EXPECT_TRUE(check_test_image(output_file));
 }

src/tests/data/color_table.yaml

@@ -0,0 +1,7 @@
+color_table: 
+  control_points: 
+    r: [0.23, 0.48, 0.99]
+    g: [0.08, 0.23, 1.0]
+    b: [0.08, 0.04, 0.9]
+    a: [1.0, 1.0, 1.0]
+    position: [0.0, 0.5, 1.0]