reading line-up corpus

Author: machisuji

src/main/java/lineup/CorpusFormatException.java

@@ -0,0 +1,12 @@
+package lineup;
+
+/**
+ * Thrown if the format of a corpus being read is invalid.
+ *
+ * @author Markus Kahl
+ */
+public class CorpusFormatException extends Exception {
+    public CorpusFormatException(String msg) {
+        super(msg);
+    }
+}

src/main/java/lineup/CorpusReader.java

@@ -0,0 +1,13 @@
+package lineup;
+
+import java.io.Reader;
+import java.util.List;
+
+/**
+ * Reads a sentence-aligned corpus.
+ *
+ * @author Markus Kahl
+ */
+public interface CorpusReader {
+    List<Translation> readCorpus(Reader reader);
+}

src/main/java/lineup/LineupCorpusReader.java

@@ -0,0 +1,108 @@
+package lineup;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Description goes here.
+ *
+ * @author Markus Kahl
+ */
+public class LineupCorpusReader implements CorpusReader {
+
+    private String sourceLanguage;
+    private String targetLanguage;
+
+    public List<Translation> readCorpus(String file) throws FileNotFoundException {
+        return readCorpus(new java.io.FileReader(file));
+    }
+
+    public List<Translation> readCorpus(Reader reader) {
+        List<Translation> result = new LinkedList<Translation>();
+        BufferedReader in = new BufferedReader(reader);
+        String source = null, target = null, line;
+
+        try {
+            while ((line = in.readLine()) != null) {
+                line = line.trim();
+                if (line.startsWith("#") || line.isEmpty()) continue;
+
+                if (source == null) {
+                    source = line;
+                } else if (target == null) {
+                    target = line;
+                }
+                if (source != null && target != null) {
+                    parseLanguage(source, true);
+                    parseLanguage(target, false);
+
+                    Translation trans = new Translation(getSourceLanguage(), getTargetLanguage());
+                    trans.getSourceSentences().addAll(parseSentences(source));
+                    trans.getTargetSentences().addAll(parseSentences(target));
+
+                    result.add(trans);
+
+                    source = null;
+                    target = null;
+                }
+            }
+        } catch (IOException e) {
+            System.err.println("Could not read corpus: " + e.getMessage() +
+                    " (" + e.getClass().getName() + ")");
+        } catch (CorpusFormatException e) {
+            System.err.println("Invalid corpus format: " + e.getMessage());
+        }
+
+        return result;
+    }
+
+    protected void parseLanguage(String line, boolean source) throws CorpusFormatException {
+        if (line.matches("[a-z]{2} \\d.*")) {
+            String lang = line.substring(0, line.indexOf(" "));
+            if (source) {
+                if (getSourceLanguage() == null) {
+                    sourceLanguage = lang;
+                } else if (!lang.equals(getSourceLanguage())) {
+                    throw new CorpusFormatException("Unexpected source language (expected "
+                            + getSourceLanguage() + ") in: " + line);
+                }
+            } else if (!source) {
+                if (getTargetLanguage() == null) {
+                    targetLanguage = lang;
+                } else if (!lang.equals(getTargetLanguage())) {
+                    throw new CorpusFormatException("Unexpected target language (expected "
+                            + getTargetLanguage() + ") in: " + line);
+                } else if (getTargetLanguage().equals(getSourceLanguage())) {
+                    throw new CorpusFormatException("Source and target language must be different.");
+                }
+            }
+        } else {
+            throw new CorpusFormatException("Expected language, got: " + line);
+        }
+    }
+
+    protected List<String> parseSentences(String line) throws CorpusFormatException {
+        String[] tokens = line.split(":|\\|{2}]");
+        if (tokens.length >= 2 && tokens[0].matches("[a-z]{2} \\d")) {
+            List<String> result = new LinkedList<String>();
+            for (int i = 1; i < tokens.length; ++i) {
+                result.add(tokens[i].trim());
+            }
+            return result;
+        } else {
+            throw new CorpusFormatException("Could not parse sentences: " + line);
+        }
+    }
+
+    public String getSourceLanguage() {
+        return sourceLanguage;
+    }
+
+    public String getTargetLanguage() {
+        return targetLanguage;
+    }
+}

src/main/java/lineup/Translation.java

@@ -0,0 +1,76 @@
+package lineup;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * A translation maps one or more sentences in one language to one or more
+ * sentences in another languages. Those sentences express the same idea.
+ *
+ * @author Markus Kahl
+ */
+public class Translation {
+
+    private String sourceLanguage;
+    private String targetLanguage;
+
+    private List<String> sourceSentences = new LinkedList<String>();
+    private List<String> targetSentences = new LinkedList<String>();
+
+    public Translation(String sourceLanguage, String targetLanguage) {
+        this.sourceLanguage = sourceLanguage;
+        this.targetLanguage = targetLanguage;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder("Translation(");
+        String src = getSourceSentences().get(0);
+        String tgt = getTargetSentences().get(0);
+        final int maxLength = 20;
+
+        sb.append(getSourceLanguage());
+        sb.append(": ");
+        if (src.length() > maxLength) {
+            sb.append(src.substring(0, maxLength - 1));
+            sb.append("...");
+        } else {
+            sb.append(src);
+            if (getSourceSentences().size() > 1 && src.length() <= maxLength) {
+                sb.append(" ...");
+            }
+        }
+
+        sb.append(" | ");
+        sb.append(getTargetLanguage());
+        sb.append(": ");
+        if (tgt.length() > maxLength) {
+            sb.append(tgt.substring(0, maxLength - 1));
+            sb.append("...");
+        } else {
+            sb.append(tgt);
+            if (getTargetSentences().size() > 1 && tgt.length() <= maxLength) {
+                sb.append(" ...");
+            }
+        }
+        sb.append(")");
+
+        return sb.toString();
+    }
+
+    public String getSourceLanguage() {
+        return sourceLanguage;
+    }
+
+    public String getTargetLanguage() {
+        return targetLanguage;
+    }
+
+    public List<String> getSourceSentences() {
+        return sourceSentences;
+    }
+
+    public List<String> getTargetSentences() {
+        return targetSentences;
+    }
+}