started script for Europarl3 corpus download

Author: machisuji

.gitignore

@@ -0,0 +1,3 @@
+.gradle
+build
+target

README.md

@@ -1,3 +1,7 @@
 # line-up
 
 A Java library for the alignment of interlinear texts.
+
+## Build
+
+SBT 0.11.3

build.sbt

@@ -0,0 +1,7 @@
+name := "line-up"
+
+version := "0.1"
+
+scalaVersion := "2.10.0"
+
+libraryDependencies += "org.scalaj" % "scalaj-http_2.10" % "0.3.7"

src/main/java/lineup/Test.java

@@ -0,0 +1,9 @@
+package lineup;
+
+import static java.lang.System.out;
+
+public class Test {
+    public static void main(String[] args) {
+        out.println("Hallo Welt");
+    }
+}

src/main/scala/opus-europarl3.scala

@@ -0,0 +1,28 @@
+package opus
+
+import scalaj.http.Http
+import sys.process._
+
+object Europarl3 {
+  val link = "<a[^/]*</a>".r
+  val href = "\"([^\"]*)\"".r
+
+  def url(lang: String) = "http://opus.lingfil.uu.se/Europarl3/xml/%s/" format lang 
+  def markup(lang: String) = Http(url(lang)).asString
+  def fetchFiles(lang: String) = link.findAllIn(markup(lang)).
+    filter(_.contains(".xml.gz")).flatMap(link => href.findAllIn(link).map(_.replace("\"", "")))
+
+  object files {
+    lazy val de = fetchFiles("de").toList
+    lazy val en = fetchFiles("en").toList
+
+    lazy val de_en = de intersect en
+  }
+
+  def download() {
+    files.de_en.take(3).foreach { file =>
+      Seq("de", "en").foreach(lang =>
+        "wget -q %s%s -O %s".format(url(lang), file, file.replace(".xml.gz", s".$lang.xml.gz")).!)
+    }
+  }
+}