Take iterator into account in query translation

morph-base/.gitignore

@@ -0,0 +1,2 @@
+/.cache-main
+/.cache-tests

morph-base/src/main/scala/es/upm/fi/dia/oeg/morph/base/engine/MorphBaseDataTranslator.scala

@@ -22,7 +22,7 @@ abstract class MorphBaseDataTranslator(val factory: IMorphFactory) {
     val logger = Logger.getLogger(this.getClass().getName());
 
     /**
-     * Entry point of the materialization approach:
+     * Entry point of the materialization approach:<br>
      * Loop on all triples maps of the mapping graph and generate triples in the data materializer model,
      * based on the triples map: this consists in calculating the query, running it against the database,
      * translating results in RDF terms and making the triples.

morph-base/src/main/scala/es/upm/fi/dia/oeg/morph/base/querytranslator/MorphBaseTriplePatternBinder.scala

@@ -28,10 +28,12 @@ import es.upm.fi.dia.oeg.morph.base.engine.IMorphFactory
 /**
  * Representation of the bindings of several triples map to a triple pattern
  */
-class TPBindings(val tp: Triple, val bound: List[R2RMLTriplesMap]) {
-
+class TPBindings(
+        val tp: Triple,
+        val bound: List[R2RMLTriplesMap]) {
+
     override def toString = { "Binding(" + tp + " -> " + bound.mkString(", ") + ")" }
-    
+
 }
 
 object TPBindings {
@@ -67,7 +69,8 @@ class MorphBaseTriplePatternBinder(factory: IMorphFactory) {
      * Compute the triple pattern bindings for all triple patterns in a graph pattern
      *
      * @param op SPARQL query or SPARQL query element
-     * @return bindings as a map of triples and associated triples maps
+     * @return bindings as a map of triples and associated triples maps.
+     * The map key is the string representation of the triple.
      */
     def bindm(op: Op): Map[String, TPBindings] = {
 
@@ -251,7 +254,7 @@ class MorphBaseTriplePatternBinder(factory: IMorphFactory) {
             }
 
             case opFilter: OpFilter => { //FILTER pattern
-                bindm(opFilter.getSubOp())                
+                bindm(opFilter.getSubOp())
             }
             case opSlice: OpSlice => {
                 Map.empty

morph-core/.gitignore

@@ -0,0 +1,2 @@
+/.cache-main
+/.cache-tests

morph-xr2rml-dist/.gitignore

@@ -0,0 +1 @@
+/.cache-main

morph-xr2rml-dist/example_mongo_rewriting/mapping.ttl

@@ -8,19 +8,15 @@
    xrr:logicalSource [ 
       xrr:query "db.movies2.find({'name':{$exists:true}})";
    ];
-
    rr:subjectMap [ rr:template "http://example.org/movie/{$.code}" ];
-
    rr:predicateObjectMap [
       rr:predicate ex:starring;
       rr:objectMap [ xrr:reference "$.actors.*" ];
    ].
 
 <#Directors>
    xrr:logicalSource [ xrr:query "db.directors2.find({})" ];
-
    rr:subjectMap [ rr:template "http://example.org/dir/{$.name}" ];
-
    rr:predicateObjectMap [
       rr:predicate ex:directed;
       rr:objectMap [
@@ -31,3 +27,17 @@
          ]
       ]
    ].
+
+# There is no data in the DB that matches this triples map, but this helps test 
+# the management of iterators in the query translation
+<#MoviesIter>
+   xrr:logicalSource [ 
+      xrr:query "db.movies2.find({'name':{$exists:true}})";
+      rml:iterator "$.movies.*"
+   ];
+   rr:subjectMap [ rr:template "http://example.org/movie/{$.code}" ];
+   rr:predicateObjectMap [
+      rr:predicate ex:starringIter;
+      rr:objectMap [ xrr:reference "$.actors.*" ];
+   ].
+

morph-xr2rml-dist/example_mongo_rewriting/query.sparql

@@ -7,8 +7,13 @@ WHERE {
 
     ?x ex:directed ?y .
     ?y ex:starring ?z .
+
+    #-- Test of rewriting with an iterator
+    #?y ex:starringIter ?z .
 
 	#<http://example.org/dir/Wong%20Kar-wai> ex:directed ?x .
 
     #?x ex:directed <http://example.org/movie/Manh> .
+
+
 }

morph-xr2rml-dist/example_taxref_rewriting/query.sparql

@@ -25,22 +25,32 @@ WHERE {
     #<http://inpn.mnhn.fr/taxref/8.0/taxon/60585> skosxl:prefLabel ?p .
     #OPTIONAL { <http://inpn.mnhn.fr/taxref/8.0/taxon/60585> skosxl:altLabel ?a . }
 
-	## Very big inner join
+	## Very big inner join but with propagation of conditions
+    #?t skosxl:prefLabel <http://inpn.mnhn.fr/taxref/label/60585> .
     #?t skosxl:altLabel ?a .
  	#?t skosxl:altLabel ?b . 
     #FILTER (?a != ?b)
 
-	## Inner join with impossible self-join elimination
+	## Big inner join: no self-join elimination nor condition propagation
     #?t skosxl:altLabel <http://inpn.mnhn.fr/taxref/label/60587> .
  	#?t skosxl:altLabel ?b . 
     #FILTER (?b != <http://inpn.mnhn.fr/taxref/label/60587>)
 
-	## Optional Self-Join elimination
+	## Optional join
+    #?t skosxl:altLabel <http://inpn.mnhn.fr/taxref/label/60587> .
+    #OPTIONAL { 
+    #  ?t skosxl:altLabel ?b .
+    #  FILTER (?b != <http://inpn.mnhn.fr/taxref/label/60587>)
+    #}
+
+	## Big optional Join - no optimization possible
     #?t skosxl:altLabel ?a .
-    #OPTIONAL { ?t skosxl:altLabel ?b . }
-    #FILTER (?a != ?b)
+    #OPTIONAL { 
+    #  ?t skosxl:altLabel ?b .
+    #  FILTER (?a != ?b)
+    #}
 
-	## Self-Join and Self-Union elimination
+	## Self-Join and Self-Union elimination, ~ 66s
     ?t skosxl:prefLabel ?p .
     ?t taxrefprop:bioGeoStatusIn ?bgs . 
     ?bgs dct:spatial <http://sws.geonames.org/3424932/> . # Saint-Pierre-Et-Miquelon

morph-xr2rml-lang/.gitignore

@@ -0,0 +1 @@
+/.cache-main

morph-xr2rml-mongo/.gitignore

@@ -0,0 +1,2 @@
+/.cache-main
+/.cache-tests

morph-xr2rml-mongo/src/main/scala/fr/unice/i3s/morph/xr2rml/mongo/abstractquery/AbstractAtomicQuery.scala

@@ -94,17 +94,14 @@ class AbstractAtomicQuery(
     }
 
     /**
-     * Translate an atomic abstract query into one or several concrete queries whose results must be UNIONed.
-     *
-     * First, the atomic abstract query is translated into an abstract MongoDB query using the
-     * JsonPathToMongoTranslator.trans() function.<br>
-     * Then, the abstract MongoDB query is translated into a set of concrete MongoDB queries
-     * by function mongoAbstractQuerytoConcrete().
-     *
-     * The result is stored in attribute this.targetQuery.
+     * Translate an atomic abstract query into one or several concrete queries whose results must be UNIONed:<br>
+     * 1. the Where part of the atomic abstract query is translated into an abstract 
+     * MongoDB query using function JsonPathToMongoTranslator.trans().<br>
+     * 2. the abstract MongoDB query is optimized and translated into a set of concrete MongoDB queries.<br>
+     * 
+     * The result is stored in attribute 'this.targetQuery'.
      *
      * @param translator the query translator
-     * @return none, the result is stored in attribute this.targetQuery.
      */
     override def translateAtomicAbstactQueriesToConcrete(translator: MorphBaseQueryTranslator): Unit = {
         val mongQTranslator = translator.asInstanceOf[MorphMongoQueryTranslator]
@@ -114,20 +111,7 @@ class AbstractAtomicQuery(
 
         // Generate one abstract MongoDB query (MongoQueryNode) for each selected condition
         val mongAbsQs: Set[MongoQueryNode] = whereConds.map(cond => {
-            if (cond.hasReference) {
-                val condIRef = cond.asInstanceOf[IReference]
-                // If there is an iterator, replace the heading "$" of the JSONPath reference with the iterator path
-                val iter = this.from.docIterator
-                if (iter.isDefined)
-                    condIRef.reference = condIRef.reference.replace("$", iter.get)
-
-                //--- Translate the condition on a JSONPath reference into an abstract MongoDB query
-                JsonPathToMongoTranslator.trans(condIRef.asInstanceOf[AbstractQueryCondition])
-            } else {
-                //--- The condition type is IsNull or Or
-                // @todo if the iterator is defined, add it to references of inner conditions
-                JsonPathToMongoTranslator.trans(cond)
-            }
+            JsonPathToMongoTranslator.trans(cond, this.from.docIterator)
         })
 
         // If there are more than one query, encapsulate them under a top-level AND
@@ -456,9 +440,9 @@ class AbstractAtomicQuery(
      * If the two queries have some shared variables, then Equality and IsNotNull conditions of the right query
      * on those shared variables can be added to the conditions of the left query.
      * @example
-     * Assume we have "$.field1 AS ?x" in left and "$.field2 AS ?x" in right.
-     * If the right query has a Where condition <code>Equals($.field2, "value")</code>, then we can add a new condition
-     * to the Where conditions of the left query: <code>Equals($.field1, "value")</code>
+     * Assume we have "\$.field1 AS ?x" in left and "\$.field2 AS ?x" in right.
+     * If the right query has a Where condition <code>Equals(\$.field2, "value")</code>, then we can add a new condition
+     * to the Where conditions of the left query: <code>Equals(\$.field1, "value")</code>
      *
      * @return an optimized version of 'this', based on the query in parameter, or 'this' is no optimization was possible.
      */

morph-xr2rml-mongo/src/main/scala/fr/unice/i3s/morph/xr2rml/mongo/query/MongoQueryNodeCond.scala

@@ -9,13 +9,14 @@ abstract class MongoQueryNodeCond extends MongoQueryNode {
 }
 
 object MongoQueryNodeCondFactory {
+
     /**
-     * Create a MongoDB query condition node from a generic AbstractQueryCondition.
+     * Create a MongoDB query condition node from an abstract query condition.
      * 
-     * straight into a MongoDB condition appended after a field name.<br>
      * Example: Equals(10) is translated into the string <code>{\$eq: 10}</code> that is appended
-     * Equals and IsNotNull are terminal conditions, i.e. they can be translated
-     * after the field name.
+     * after a field name: <code>'field': {\$eq: 10}</code>.<br>
+     * <i>Equals</i> and <i>IsNotNull</i> are terminal conditions for MongoDB, i.e. they can be 
+     * translated straight after the field name.
      *  
      * Conversely, <i>IsNull</i>, <i>Or</i> and <i>And</i> conditions are non terminal for MongoDB.<br> 
      * Example: "IsNull($.field)" means that the field either does not exist or is null.
@@ -25,7 +26,10 @@ object MongoQueryNodeCondFactory {
      * <code>\$or: [{'field': {\$eq: null}}, {'field': {\$exists: false}}]</code>
      * 
      * This factory only deals with the first two cases.
-     * The two latter are handled in JsonPathToMongoTranslator.trans()
+     * The two latter are handled in JsonPathToMongoTranslator.trans().
+     * 
+     * @param cond an abstract query condition
+     * @return a MongoDB query condition node
      * 
      * @todo Condition type SparqlFilter is not managed.
      */

morph-xr2rml-mongo/src/main/scala/fr/unice/i3s/morph/xr2rml/mongo/querytranslator/JsonPathToMongoTranslator.scala

@@ -144,27 +144,35 @@ object JsonPathToMongoTranslator {
      * @param cond and condition on a JSONPath expression to translate, must NOT be empty or null
      * @return a MongoQueryNode instance representing the top-level MongoDB query.
      * The result CANNOT be null, but a MongoQueryNodeNotSupported is returned in case no rule matched.
-     *
      */
-    def trans(cond: AbstractQueryCondition): MongoQueryNode = {
-        trans(cond, List.empty)
+    def trans(cond: AbstractQueryCondition, iter: Option[String]): MongoQueryNode = {
+        trans(cond, iter, List.empty)
     }
 
     /**
-     * Entry point of the translation of abstract query condition on a JSONPath expression 
-     * into an abstract MongoDB query. The resulting query is not optimized.
+     * Entry point of the translation of an abstract query condition on a JSONPath expression
+     * into an abstract MongoDB query. The resulting abstract MongoDB query is not optimized.
      *
      * @param cond and condition on a JSONPath expression to translate, must  be empty or null
-     * @param projection set of projections to push in the MongoDB query (@todo not implemented)
+     * @param iter the iterator from the logical source, i.e. the From part of the anstract atomic query
+     * @param projection set of projections to push in the MongoDB query
+     * 
      * @return a MongoQueryNode instance representing the top-level MongoDB query.
      * The result CANNOT be null, but a MongoQueryNodeNotSupported is returned in case no rule matched.
      *
+     * @todo the translation of 'projection' into actual MongoDB projections is not managed.
      */
-    def trans(cond: AbstractQueryCondition, projection: List[MongoQueryProjection]): MongoQueryNode = {
-        var path =
+    def trans(cond: AbstractQueryCondition, iter: Option[String], projection: List[MongoQueryProjection]): MongoQueryNode = {
+
+        val path =
             if (cond.hasReference) {
                 val ref = cond.asInstanceOf[IReference].reference
-                if (ref == null) "" else ref
+                if (ref == null) "" else {
+                    // If there is an iterator, replace the heading "$" of the JSONPath reference with the iterator path
+                    if (iter.isDefined)
+                        ref.replace("$", iter.get)
+                    else ref
+                }
             } else ""
 
         cond.condType match {
@@ -180,13 +188,13 @@ object JsonPathToMongoTranslator {
             case ConditionType.Or => {
                 // AbstractQueryConditionOr => MongoQueryNodeOr
                 val condOr = cond.asInstanceOf[AbstractQueryConditionOr]
-                new MongoQueryNodeOr(condOr.members.map(c => trans(c, projection)))
+                new MongoQueryNodeOr(condOr.members.map(c => trans(c, iter, projection)))
             }
 
             case ConditionType.And => {
                 // AbstractQueryConditionAnd => MongoQueryNodeAnd
                 val condAnd = cond.asInstanceOf[AbstractQueryConditionAnd]
-                new MongoQueryNodeAnd(condAnd.members.map(c => trans(c, projection)))
+                new MongoQueryNodeAnd(condAnd.members.map(c => trans(c, iter, projection)))
             }
 
             case _ =>

morph-xr2rml-mongo/src/main/scala/fr/unice/i3s/morph/xr2rml/mongo/querytranslator/MorphMongoQueryTranslator.scala

@@ -57,14 +57,11 @@ class MorphMongoQueryTranslator(factory: IMorphFactory) extends MorphBaseQueryTr
     /**
      * High level entry point to the query translation process.
      *
-     * @todo Several features are not implemented:<br>
-     * - The project part is calculated but not managed: all fields of MongoDB documents are retrieved.
+     * @todo The project part is calculated but not managed: all fields of MongoDB documents are retrieved.
      * Besides, only the references are listed, but they must be bound to the variable they represent
      * (the AS of the project part) so that the INNER JOIN can be computed, and from the reference
      * we must figure out which field exactly is to be projected and translate this into a MongoDB
-     * collection.find() projection parameter. E.g.: $.field.* =&gt; {'field':true}<br>
-     * - The iterator of the logical source is not taken into account when computing the WHERE part, i.e.
-     * the conditions on the JSONPath references from the mapping.
+     * collection.find() projection parameter. E.g.: \$.field.* =&gt; {'field':true}<br>
      *
      * @return a AbstractQuery instance in which the targetQuery parameter has been set with
      * a list containing a set of concrete queries. May return None if no bindings are found.
@@ -128,15 +125,12 @@ class MorphMongoQueryTranslator(factory: IMorphFactory) extends MorphBaseQueryTr
                         Some(this.transTPm(tpBindings.get))
                 } else {
                     // Make an INER JOIN between the first triple pattern and the rest of the triple patterns
-
                     val tpBindingsLeft = this.getTpBindings(bindings, triples.head)
                     if (!tpBindingsLeft.isDefined) {
                         logger.warn("No binding defined for triple pattern " + triples.head.toString)
                         None
                     } else {
-
                         val left = this.transTPm(tpBindingsLeft.get)
-
                         val right = this.translateSparqlQuery(bindings, new OpBGP(BasicPattern.wrap(triples.tail)))
                         if (right.isDefined)
                             Some(AbstractQueryInnerJoin(left, right.get))
@@ -284,18 +278,19 @@ class MorphMongoQueryTranslator(factory: IMorphFactory) extends MorphBaseQueryTr
     }
 
     /**
-     * Translate a non-optimized MongoQueryNode instance into one or more optimized concrete MongoDB queries.
+     * Translate a non-optimized MongoQueryNode instance into one or more optimized concrete MongoDB queries
+     * whose results must be UNIONed.
      *
      * Firstly, the query is optimized, which may generate a top-level UNION.
      * Then, a query string is generated from the optimized MongoQueryNode, to which the initial query string
-     * from the logical source is appended.
+     * (from the logical source) is appended.
      * A UNION is translated into several concrete queries. For any other type of query only one query string is returned.
      *
      * @todo the project part is not managed: must transform JSONPath references into actually projectable
      * fields in a MongoDB query
      *
-     * @param from from part of the atomic abstract query (query from the logical source)
-     * @param project project part of the atomic abstract query (xR2RML references to project, NOT MANAGED FOR NOW).
+     * @param from From part of the atomic abstract query (query from the logical source)
+     * @param project Project part of the atomic abstract query (xR2RML references to project, NOT MANAGED FOR NOW).
      * @param absQuery the abstract MongoDB query to translate into concrete MongoDB queries.
      * @return list of MongoDBQuery instances. If there are several instances, their results must be UNIONed.
      */
@@ -308,7 +303,7 @@ class MorphMongoQueryTranslator(factory: IMorphFactory) extends MorphBaseQueryTr
         if (logger.isTraceEnabled())
             logger.trace("Condtions optimized to: " + Q)
 
-        // If the query is an AND, merge its FIELD nodes that have the same path
+        // If the query is an AND, merge its FIELD nodes that have the same root path
         // Example 1. AND('a':{$gt:10}, 'a':{$lt:20}) => 'a':{$gt:10, $lt:20}
         // Example 2. AND('a.b':{$elemMatch:{Q1}}, 'a.b':{$elemMatch:{Q2}}) => 'a.b': {$elemMatch:{$and:[{Q1},{Q2}]}}.
         if (Q.isAnd)