libgfapi-java-io / native hadoop shim for glusterfs.

pom.xml

@@ -9,7 +9,40 @@
   <version>2.1.6</version>
   <name>glusterfs-hadoop</name>
   <url>http://maven.apache.org</url>
+
+   <!--  include the maven repo for the libgf-java-io native bindings -->
+  <repositories>  
+     <repository>  
+       <id>libgfapi-java-release-repo</id>  
+       <name>libgfapi-java Release Repository</name>  
+       <url>http://snuffy.s3-website-us-east-1.amazonaws.com/release</url>  
+       <releases>  
+         <enabled>true</enabled>  
+       </releases>  
+       <snapshots>  
+         <enabled>false</enabled>  
+       </snapshots>  
+     </repository>  
+     <repository>  
+       <id>libgfapi-java-s3-snapshot-repo</id>  
+       <name>libgfapi-java Snapshot Repository</name>  
+       <url>http://snuffy.s3-website-us-east-1.amazonaws.com/snapshot</url>  
+       <releases>  
+         <enabled>false</enabled>  
+       </releases>  
+       <snapshots>  
+         <enabled>true</enabled>  
+       </snapshots>  
+     </repository>  
+   </repositories>  
+
+
   <dependencies>
+    <dependency>
+	<groupId>glusterfs</groupId>
+		<artifactId>libgfapi-java-io</artifactId>
+		<version>1.0-SNAPSHOT</version>
+	</dependency>
     <dependency>
 	<groupId>junit</groupId>
 		<artifactId>junit</artifactId>

src/main/java/org/apache/hadoop/fs/libgfsio/GlusterFSBrickClass.java

@@ -0,0 +1,104 @@
+/**
+ *
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.libgfsio;
+
+import java.io.*;
+
+public class GlusterFSBrickClass{
+    String host;
+    String exportedFile;
+    long start;
+    long end;
+    boolean isChunked;
+    int stripeSize; // Stripe size in bytes
+    int nrStripes; // number of stripes
+    int switchCount; // for SR, DSR - number of replicas of each stripe
+                     // -1 for others
+
+    public GlusterFSBrickClass(String brick, long start, long len, boolean flag, int stripeSize, int nrStripes, int switchCount) throws IOException{
+        this.host=brick2host(brick);
+        this.exportedFile=brick2file(brick);
+        this.start=start;
+        this.end=start+len;
+        this.isChunked=flag;
+        this.stripeSize=stripeSize;
+        this.nrStripes=nrStripes;
+        this.switchCount=switchCount;
+    }
+
+    public boolean isChunked(){
+        return isChunked;
+    }
+
+    public String brickIsLocal(String hostname){
+        String path=null;
+        if(host.equals(hostname))
+            path=exportedFile;
+
+        return path;
+    }
+
+    public int[] getBrickNumberInTree(long start,int len){
+        long end=len;
+        int startNodeInTree=((int) (start/stripeSize))%nrStripes;
+        int endNodeInTree=((int) ((start+len)/stripeSize))%nrStripes;
+
+        if(startNodeInTree!=endNodeInTree){
+            end=(start-(start%stripeSize))+stripeSize;
+            end-=start;
+        }
+
+        return new int[]{startNodeInTree,endNodeInTree,(int) end};
+    }
+
+    public boolean brickHasFilePart(int nodeInTree,int nodeLoc){
+        if(switchCount==-1)
+            return (nodeInTree==nodeLoc);
+
+        nodeInTree*=switchCount;
+        for(int i=nodeInTree;i<(nodeInTree+switchCount);i++){
+            if(i==nodeLoc)
+                return true;
+        }
+
+        return false;
+    }
+
+    public String brick2host(String brick) throws IOException{
+        String[] hf=null;
+
+        hf=brick.split(":");
+        if(hf.length!=2)
+            throw new IOException("Error getting hostname from brick");
+
+        return hf[0];
+    }
+
+    public String brick2file(String brick) throws IOException{
+        String[] hf=null;
+
+        hf=brick.split(":");
+        if(hf.length!=2)
+            throw new IOException("Error getting hostname from brick");
+
+        return hf[1];
+    }
+
+}

src/main/java/org/apache/hadoop/fs/libgfsio/GlusterFSBrickRepl.java

@@ -0,0 +1,50 @@
+/**
+ *
+ * Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.libgfsio;
+
+public class GlusterFSBrickRepl{
+    private String[] replHost;
+    private long start;
+    private long len;
+    private int cnt;
+
+    public GlusterFSBrickRepl(int replCount, long start, long len){
+        this.replHost=new String[replCount];
+        this.start=start;
+        this.len=len;
+        this.cnt=0;
+    }
+
+    public void addHost(String host){
+        this.replHost[cnt++]=host;
+    }
+
+    public String[] getReplHosts(){
+        return this.replHost;
+    }
+
+    public long getStartLen(){
+        return this.start;
+    }
+
+    public long getOffLen(){
+        return this.len;
+    }
+}

src/main/java/org/apache/hadoop/fs/libgfsio/GlusterFileStatus.java

@@ -0,0 +1,58 @@
+/**
+ *
+ * Copyright (c) 2011 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ * Implements the Hadoop FileSystem Interface to allow applications to store
+ * files on GlusterFS and run Map/Reduce jobs on the data.
+ * 
+ * 
+ */
+
+package org.apache.hadoop.fs.libgfsio;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
+import org.gluster.fs.GlusterFile;
+/*
+ * Copied from org.apache.fs.RawLocalFileSystem.RawFileStatus
+ */
+public class GlusterFileStatus extends FileStatus{
+
+
+
+	public GlusterFileStatus(GlusterFile file, GlusterfsVolume vol) { 
+		  super(file.length(),
+				file.isDirectory(),
+				0, // repliation count
+				file.getBlockSize(),
+				file.getMtime(),
+				file.getAtime(),
+				new FsPermission(new Long(file.getMod()).shortValue()),
+				IdLookup.getName(new Long(file.getUid()).intValue()),
+				IdLookup.getName(new Long(file.getGid()).intValue()),
+				null, // path to link 
+				vol.makeQualified(new Path(file.getPath())));
+
+	  }
+}

src/main/java/org/apache/hadoop/fs/libgfsio/GlusterFileSystem.java

@@ -0,0 +1,89 @@
+/**
+ *
+ * Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+/**
+ * Implements the Hadoop FileSystem Interface to allow applications to store
+ * files on GlusterFS and run Map/Reduce jobs on the data.  This code does NOT perform a CRC 
+ * on files.
+ * 
+ * gluster file systems are specified with the glusterfs:// prefix.
+ * 
+ * 
+ */
+package org.apache.hadoop.fs.libgfsio;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GlusterFileSystem extends FilterFileSystem{
+
+    protected static final Logger log=LoggerFactory.getLogger(GlusterFileSystem.class);
+
+    public GlusterFileSystem(){
+        super(new GlusterfsVolume());
+        Version v=new Version();
+        log.info("Initializing GlusterFS,  CRC disabled.");
+        log.info("GIT INFO="+v);
+        log.info("GIT_TAG="+v.getTag());
+    }
+
+    /**
+     * Get file status.
+     */
+    public boolean exists(Path f) throws IOException{
+    	   return getRawFileSystem().exists(f);
+    }
+
+    public void setConf(Configuration conf){
+        log.info("Configuring GlusterFS");
+        super.setConf(conf);
+    }
+
+    /*
+     * if GlusterFileSystem is the default filesystem, real local URLs come back
+     * without a file:/ scheme name (BUG!). the glusterfs file system is
+     * assumed. force a schema.
+     */
+
+    public void copyFromLocalFile(boolean delSrc,Path src,Path dst) throws IOException{
+        FileSystem srcFs=new Path("file:/"+src.toString()).getFileSystem(getConf());
+        FileSystem dstFs=dst.getFileSystem(getConf());
+        FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf());
+    }
+
+    public void copyToLocalFile(boolean delSrc,Path src,Path dst) throws IOException{
+        FileSystem srcFs=src.getFileSystem(getConf());
+        FileSystem dstFs=new Path("file:/"+dst.toString()).getFileSystem(getConf());
+        FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf());
+    }
+
+    public String toString(){
+        return "Gluster File System, no CRC.";
+    }
+}

src/main/java/org/apache/hadoop/fs/libgfsio/GlusterFs.java

@@ -0,0 +1,51 @@
+/**
+ *
+ * Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * Licensed under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+/**
+ * Implements the Hadoop FileSystem 2.x Interface to allow applications to store
+ * files on GlusterFS and run Map/Reduce jobs on the data.  This code does not perform a CRC 
+ * on the files.
+ * 
+ * gluster file systems are specified with the glusterfs:// prefix.
+ * 
+ * 
+ */
+
+package org.apache.hadoop.fs.libgfsio;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FilterFs;
+
+public class GlusterFs extends FilterFs{
+
+    GlusterFs(Configuration conf) throws IOException, URISyntaxException{
+        super(new GlusterVol(conf));
+    }
+
+
+    GlusterFs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException{
+        this(conf);
+    }
+
+
+}