Use Spotlight core tokenizer when collecting ngrams (introduces dependecy to Spotlight Core).

Author: jodaiber

examples/indexing/names_and_entities.pig

@@ -118,7 +118,7 @@ sfs = UNION ONSCHEMA
 STORE sfs INTO '$TEMPORARY_SF_LOCATION/sfs';
 
 -- Define Ngram generator with maximum Ngram length
-DEFINE ngramGenerator pignlproc.helpers.RestrictedNGramGenerator('$MAX_NGRAM_LENGTH', '$TEMPORARY_SF_LOCATION/sfs');
+DEFINE ngramGenerator pignlproc.helpers.RestrictedNGramGenerator('$MAX_NGRAM_LENGTH', '$TEMPORARY_SF_LOCATION/sfs', '$LOCALE');
 
 EXEC;
 

examples/indexing/names_and_entities.pig.params

@@ -3,7 +3,8 @@ PIGNLPROC_JAR=%PIG_PATH/target/pignlproc-0.1.0-SNAPSHOT.jar
 # number of reducers
 DEFAULT_PARALLEL=6
 
-LANG=nl
+LANG=%LANG
+LOCALE=%LOCALE
 
 INPUT=/user/hadoop/%LANGwiki-latest-pages-articles.xml
 OUTPUT=hdfs:///user/hadoop/%LANG/names_and_entities

pom.xml

@@ -89,6 +89,12 @@
       <version>4.2</version>
       <scope>test</scope>
     </dependency>
+      <dependency>
+          <groupId>org.dbpedia.spotlight</groupId>
+          <version>0.6</version>
+          <artifactId>core</artifactId>
+      </dependency>
+
   </dependencies>
 
   <build>

src/main/java/pignlproc/helpers/RestrictedNGramGenerator.java

@@ -18,23 +18,21 @@
  * limitations under the License.
  */
 
-import opennlp.tools.tokenize.SimpleTokenizer;
-import opennlp.tools.tokenize.Tokenizer;
 import opennlp.tools.util.Span;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.FuncSpec;
 import org.apache.pig.data.*;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.dbpedia.spotlight.db.model.RawTokenizer;
+import org.dbpedia.spotlight.db.model.Stemmer;
+import org.dbpedia.spotlight.db.tokenize.LanguageIndependentRawTokenizer;
 
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 
 
 /**
@@ -47,7 +45,7 @@ public class RestrictedNGramGenerator extends EvalFunc<DataBag> {
 
     private int ngramSizeLimit;
 
-    private final Tokenizer tokenizer = SimpleTokenizer.INSTANCE;
+    private RawTokenizer tokenizer;
 
     private final BagFactory bagFactory = DefaultBagFactory.getInstance();
     private final TupleFactory tupleFactory = TupleFactory.getInstance();
@@ -56,14 +54,17 @@ public class RestrictedNGramGenerator extends EvalFunc<DataBag> {
     private String surfaceFormListFile;
 
 
-    public RestrictedNGramGenerator(int ngramSizeLimit, String surfaceFormListFile) {
+    public RestrictedNGramGenerator(int ngramSizeLimit, String surfaceFormListFile, String locale) {
         this.ngramSizeLimit = ngramSizeLimit;
         this.surfaceFormListFile = surfaceFormListFile;
+
+        String[] localeA = locale.split("_");
+        this.tokenizer = new LanguageIndependentRawTokenizer(new Locale(localeA[0], localeA[1]), new Stemmer());
     }
 
     // Pig versions < 0.9 seem to only pass strings in constructor
-    public RestrictedNGramGenerator(String ngramSizeLimit, String surfaceFormListFile) {
-        this(Integer.valueOf(ngramSizeLimit), surfaceFormListFile);
+    public RestrictedNGramGenerator(String ngramSizeLimit, String surfaceFormListFile, String locale) {
+        this(Integer.valueOf(ngramSizeLimit), surfaceFormListFile, locale);
     }
 
     public List<String> getCacheFiles() {