From 58766951c524684cdec59da30c3e82b1ffc15526 Mon Sep 17 00:00:00 2001 From: Timothy St. Clair Date: May 28 2014 20:20:27 +0000 Subject: Update build --- diff --git a/amplab-tachyon.spec b/amplab-tachyon.spec index b23c096..a5d79a1 100644 --- a/amplab-tachyon.spec +++ b/amplab-tachyon.spec @@ -1,9 +1,9 @@ -%global commit 71028e3478eac70b20d76ad75ae4fe771b2515da +%global commit fc8d2fdd3f49738216619babaf85a4bdcd56dbe8 %global shortcommit %(c=%{commit}; echo ${c:0:7}) %global shortname tachyon Name: amplab-%{shortname} -# Given the naming conflicts with other packages, and eventually this will +# Given the naming conflicts with other packages, and eventually this will # switch to apache-tachyon should Version: 0.5.0 Release: 1.SNAPSHOT.%{shortcommit}%{?dist} @@ -19,11 +19,11 @@ Source5: %{shortname}-env.sh Patch0: tachyon-0.4.0-SNAPSHOT-log4props.patch Patch1: tachyon-0.4.0-defaults.patch -Patch2: tachyon-0.5.0-gluster.patch BuildRequires: java-devel BuildRequires: mvn(commons-io:commons-io) -BuildRequires: mvn(log4j:log4j) + +BuildRequires: log4j12 BuildRequires: mvn(org.apache.ant:ant) BuildRequires: mvn(org.apache.commons:commons-lang3) @@ -85,7 +85,6 @@ find -name '*.jar' -print -delete %patch0 -F2 -p1 %patch1 -F2 -p1 -%patch2 -p1 sed -i "s|hadoop-client|hadoop-mapreduce-client-core|" pom.xml @@ -103,7 +102,7 @@ sed -i "s|hadoop-client|hadoop-mapreduce-client-core|" p %pom_add_dep org.apache.hadoop:hadoop-hdfs %build - + %mvn_file org.tachyonproject:%{shortname} %{shortname} %mvn_build @@ -144,7 +143,7 @@ mkdir -p -m0755 %{buildroot}/%{_datadir}/%{shortname}/web cp -rf main/src/main/webapp %{buildroot}/%{_datadir}/%{shortname}/web ####################### -# NOTE: The following is plugging into hadoop without +# NOTE: The following is plugging into hadoop without # having administrative steps. mkdir -p -m0755 %{buildroot}/%{_datadir}/hadoop/common/lib %{__ln_s} %{_javadir}/%{shortname}.jar %{buildroot}/%{_datadir}/hadoop/common/lib @@ -190,7 +189,7 @@ mkdir -p -m0755 %{buildroot}/%{_datadir}/hadoop/common/lib %systemd_postun_with_restart %{shortname}-slave.service %{shortname}-master.service %changelog -* Mon May 5 2014 Timothy St. Clair - 0.5.0-1.SNAPSHOT.71028e3 +* Wed May 28 2014 Timothy St. Clair - 0.5.0-1.SNAPSHOT.fc8d2fd - Update to 0.5.0 * Mon Feb 24 2014 Timothy St. Clair - 0.4.1-2.SNAPSHOT.4b9c806 @@ -218,7 +217,7 @@ mkdir -p -m0755 %{buildroot}/%{_datadir}/hadoop/common/lib - Pre-release update to 0.4.0 with script modifications. * Thu Oct 10 2013 Timothy St. Clair 0.3.0-1 -- Update to the latest in preparation for release. +- Update to the latest in preparation for release. * Sun Sep 29 2013 gil cattaneo 0.2.1-1 - initial rpm diff --git a/sources b/sources index 3a3febd..ed5574f 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -e5c4849a38908e408a30073b2f67bbaf tachyon-0.5.0-71028e3.tar.gz +2ea3541e3778661379d57b8f7eb67948 tachyon-0.5.0-fc8d2fd.tar.gz diff --git a/tachyon-0.5.0-gluster.patch b/tachyon-0.5.0-gluster.patch deleted file mode 100644 index ac15684..0000000 --- a/tachyon-0.5.0-gluster.patch +++ /dev/null @@ -1,2513 +0,0 @@ -diff --git a/bin/tachyon b/bin/tachyon -index d062f2b..9aea6c8 100755 ---- a/bin/tachyon -+++ b/bin/tachyon -@@ -131,7 +131,7 @@ PARAMETER="" - if [ "$COMMAND" == "format" ]; then - if [ $# -eq 1 ]; then - if [ "$1" == "-s" ]; then -- if [ -e $TACHYON_UNDERFS_ADDRESS ] || [[ $TACHYON_UNDERFS_ADDRESS == hdfs://* ]] || [[ $TACHYON_UNDERFS_ADDRESS == s3://* ]]; then -+ if [ -e $TACHYON_UNDERFS_ADDRESS ] || [[ $TACHYON_UNDERFS_ADDRESS == hdfs://* ]] || [[ $TACHYON_UNDERFS_ADDRESS == s3://* ]] || [[ $TACHYON_UNDERFS_ADDRESS == glusterfs://* ]]; then - # already exists, hdfs, or s3, don't format - exit 0 - else -diff --git a/conf/tachyon-glusterfs-env.sh.template b/conf/tachyon-glusterfs-env.sh.template -new file mode 100755 -index 0000000..325246e ---- /dev/null -+++ b/conf/tachyon-glusterfs-env.sh.template -@@ -0,0 +1,57 @@ -+#!/usr/bin/env bash -+ -+# This file contains environment variables required to run Tachyon. Copy it as tachyon-env.sh and -+# edit that to configure Tachyon for your site. At a minimum, -+# the following variables should be set: -+# -+# - JAVA_HOME, to point to your JAVA installation -+# - TACHYON_MASTER_ADDRESS, to bind the master to a different IP address or hostname -+# - TACHYON_UNDERFS_ADDRESS, to set the under filesystem address. -+# - TACHYON_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker -+# - TACHYON_RAM_FOLDER, to set where worker stores in memory data -+# - TACHYON_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem, -+# org.apache.hadoop.hdfs.DistributedFileSystem) -+ -+# The following gives an example: -+ -+if [[ `uname -a` == Darwin* ]]; then -+ # Assuming Mac OS X -+ export JAVA_HOME=$(/usr/libexec/java_home) -+ export TACHYON_RAM_FOLDER=/Volumes/ramdisk -+ export TACHYON_JAVA_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc=" -+else -+ # Assuming Linux -+ if [ -z "$JAVA_HOME" ]; then -+ export JAVA_HOME=/usr/lib/jvm/java-1.6.0-sun-1.6.0.45.x86_64/ -+ fi -+ export TACHYON_RAM_FOLDER=/mnt/ramdisk -+fi -+ -+export TACHYON_MASTER_ADDRESS=localhost -+ -+export TACHYON_UNDERFS_ADDRESS=glusterfs:/// -+export TACHYON_UNDERFS_GLUSTER_VOLUMES=tachyon_vol -+export TACHYON_UNDERFS_GLUSTER_MOUNTS=/vol -+export TACHYON_UNDERFS_GLUSTER_MR_DIR=glusterfs:///mapred/system -+export TACHYON_WORKER_MEMORY_SIZE=1GB -+ -+CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -+ -+export TACHYON_JAVA_OPTS+=" -+ -Dlog4j.configuration=file:$CONF_DIR/log4j.properties -+ -Dtachyon.debug=true -+ -Dtachyon.underfs.address=$TACHYON_UNDERFS_ADDRESS -+ -Dtachyon.data.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/data -+ -Dtachyon.workers.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/workers -+ -Dtachyon.worker.memory.size=$TACHYON_WORKER_MEMORY_SIZE -+ -Dtachyon.worker.data.folder=$TACHYON_RAM_FOLDER/tachyonworker/ -+ -Dtachyon.master.worker.timeout.ms=60000 -+ -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS -+ -Dtachyon.master.journal.folder=$TACHYON_HOME/journal/ -+ -Dtachyon.master.pinlist=/pinfiles;/pindata -+ -Dtachyon.underfs.glusterfs.volumes=$TACHYON_UNDERFS_GLUSTER_VOLUMES -+ -Dtachyon.underfs.glusterfs.mounts=$TACHYON_UNDERFS_GLUSTER_MOUNTS -+ -Dtachyon.underfs.glusterfs.mapred.system.dir=$TACHYON_UNDERFS_GLUSTER_MR_DIR -+ -Dorg.apache.jasper.compiler.disablejsr199=true -+" -+export JAVA="$JAVA_HOME/bin/java $TACHYON_JAVA_OPTS" -\ No newline at end of file -diff --git a/docs/Startup-Tasks-for-New-Contributors.md b/docs/Startup-Tasks-for-New-Contributors.md -index fb72ba5..2e65c10 100644 ---- a/docs/Startup-Tasks-for-New-Contributors.md -+++ b/docs/Startup-Tasks-for-New-Contributors.md -@@ -46,13 +46,6 @@ You can generate Eclipse configure file by run: - - Then import the folder into Eclipse. - --### Testing -- --If you want to run unit tests, you can use command: mvn test. -- --If you want to run a single tests, you can use command: mvn -Dtest=TestCircle#mytest test ; --e.g. mvn -Dtest=TachyonFSTest#createFileTest test ; -- - ### Coding Style - - - Follow the style of the existing codebase. Specifically, we use -@@ -71,10 +64,8 @@ e.g. mvn -Dtest=TachyonFSTest#createFileTest test ; - - Break your work into small, single-purpose patches if possible. It’s much harder to merge in - a large change with a lot of disjoint features. - --- Make sure that any methods you add maintain the alphabetical ordering of method names in each file. -- - - Submit the patch as a GitHub pull request. For a tutorial, see the GitHub guides on - [forking a repo](https://help.github.com/articles/fork-a-repo) and - [sending a pull request](https://help.github.com/articles/using-pull-requests). - --- Make sure that your code passes the unit tests: mvn test. -+- Make sure that your code passes the unit tests. -diff --git a/docs/Syncing-the-Underlying-Filesystem.md b/docs/Syncing-the-Underlying-Filesystem.md -index 9fc449a..7ab9c8c 100644 ---- a/docs/Syncing-the-Underlying-Filesystem.md -+++ b/docs/Syncing-the-Underlying-Filesystem.md -@@ -3,33 +3,15 @@ layout: global - title: Syncing the Underlayer Filesystem - --- - --Often times, there is already data in the underlying store, but when Tachyon is started, it will --not have knowledge about the preexisting files. -+Often times, there is already data in the underlying store, but when Tachyon is started, it will not -+have knowledge about the preexisting files. - - Use the tachyon shell command loadufs to sync the filesystems. - -- $ ./bin/tachyon loadufs [TACHYON_PATH] [UNDERLYING_FILESYSTEM_PATH] [Optional EXCLUDE_PATHS] -+ $ ./bin/tachyon loadufs [TACHYON_ADDRESS] [UNDERLYING_FILESYSTEM_ADDRESS] [ROOT_DIRECTORY] [-Optional EXCLUDE_PATHS] - - For example: - -- $ ./bin/tachyon loadufs tachyon://127.0.0.1:19998 hdfs://localhost:9000 tachyon -+ $ ./bin/tachyon loadufs tachyon://127.0.0.1:19998 hdfs://localhost:9000 / /tachyon - --Would load the meta-data for all the files in the local hdfs, except for the tachyon folder. -- -- $ ./bin/tachyon loadufs tachyon://127.0.0.1:19998/tomlogs file:///Users/tom/logs tachyon;spark -- --Would load meta-data for all local files under the /Users/tom/logs directory (except for tachyon --and spark) to address tachyon://127.0.0.1:19998/tomlogs. If /Users/tom/logs itself is a file, only --that file is loaded as /tomlogs/logs in the TFS. The prefix "file://" can be safely omitted for --a local file system. -- --Note that the optional EXCLUDE_PATHS are prefixes relative to the given local file path. Moreover, --only files matching the given prefixes relative to the path will be excluded. Hence, in the above --last example, logs/tachyon and logs/spark will be excluded, but not logs/shark/tachyon nor --logs/shark/spark. To exclude these two paths as well, the exclude list should be specified as --"tachyon;spark;shark/tachyon;shark/spark". It is important to note that when ";" is present to --concatenate multiple prefixes the quote marks must be used; otherwise it would be treated as --multiple commands to be executed in tandem. -- --In a sense, loadufs is similar to the unix mount command. It's not called mount so as not to cause --confusions with the use of mount in the tachyon scripts. -+Would load the meta-data for all the files in the local hdfs, except for the Tachyon folder. -diff --git a/main/pom.xml b/main/pom.xml -index d5e0ea5..2db4297 100644 ---- a/main/pom.xml -+++ b/main/pom.xml -@@ -49,6 +49,13 @@ - jar - compile - -+ -+ org.apache.hadoop.fs.glusterfs -+ glusterfs-hadoop -+ 2.1.8 -+ jar -+ compile -+ - - org.eclipse.jetty - jetty-server -diff --git a/main/src/main/java/tachyon/Constants.java b/main/src/main/java/tachyon/Constants.java -index da558a8..e286d36 100644 ---- a/main/src/main/java/tachyon/Constants.java -+++ b/main/src/main/java/tachyon/Constants.java -@@ -22,7 +22,6 @@ public class Constants { - public static final int MB = KB * 1024; - public static final int GB = MB * 1024; - public static final long TB = GB * 1024L; -- public static final long PB = TB * 1024L; - - public static final String ANSI_RESET = "\u001B[0m"; - public static final String ANSI_BLACK = "\u001B[30m"; -diff --git a/main/src/main/java/tachyon/Format.java b/main/src/main/java/tachyon/Format.java -index 51ba23b..7af0819 100644 ---- a/main/src/main/java/tachyon/Format.java -+++ b/main/src/main/java/tachyon/Format.java -@@ -71,7 +71,7 @@ public class Format { - if (ufs.exists(localFolder)) { - String[] files = ufs.list(localFolder); - for (String file : files) { -- ufs.delete(CommonUtils.concat(localFolder, file), true); -+ ufs.delete(localFolder + Constants.PATH_SEPARATOR + file, true); - } - } - } else { -@@ -79,4 +79,4 @@ public class Format { - System.exit(-1); - } - } --} -+} -\ No newline at end of file -diff --git a/main/src/main/java/tachyon/LeaderInquireClient.java b/main/src/main/java/tachyon/LeaderInquireClient.java -index defcd03..826fdc2 100644 ---- a/main/src/main/java/tachyon/LeaderInquireClient.java -+++ b/main/src/main/java/tachyon/LeaderInquireClient.java -@@ -72,7 +72,8 @@ public class LeaderInquireClient { - long maxTime = 0; - String leader = ""; - for (String master : masters) { -- Stat stat = CLIENT.checkExists().forPath(CommonUtils.concat(LEADER_PATH, master)); -+ Stat stat = -+ CLIENT.checkExists().forPath(LEADER_PATH + Constants.PATH_SEPARATOR + master); - if (stat != null && stat.getCtime() > maxTime) { - maxTime = stat.getCtime(); - leader = master; -diff --git a/main/src/main/java/tachyon/PrefixList.java b/main/src/main/java/tachyon/PrefixList.java -index 5acdd15..9528319 100644 ---- a/main/src/main/java/tachyon/PrefixList.java -+++ b/main/src/main/java/tachyon/PrefixList.java -@@ -67,18 +67,4 @@ public class PrefixList { - public boolean outList(String path) { - return !inList(path); - } -- -- /** -- * Print out all prefixes separated by ";". -- * -- * @return the string representation like "a;b/c" -- */ -- @Override -- public String toString() { -- StringBuilder s = new StringBuilder(); -- for (String prefix : LIST) { -- s.append(prefix).append(";"); -- } -- return s.toString(); -- } - } -\ No newline at end of file -diff --git a/main/src/main/java/tachyon/UnderFileSystem.java b/main/src/main/java/tachyon/UnderFileSystem.java -index 6e27464..b8e8d40 100644 ---- a/main/src/main/java/tachyon/UnderFileSystem.java -+++ b/main/src/main/java/tachyon/UnderFileSystem.java -@@ -44,49 +44,16 @@ public abstract class UnderFileSystem { - } - - public static UnderFileSystem get(String path) { -- if (path.startsWith("hdfs://") || path.startsWith("s3://") || path.startsWith("s3n://")) { -+ if (path.startsWith("hdfs://") || path.startsWith("file://") || path.startsWith("s3://") -+ || path.startsWith("s3n://") || path.startsWith("glusterfs:///")) { - return UnderFileSystemHdfs.getClient(path); -- } else if (path.startsWith(Constants.PATH_SEPARATOR) || path.startsWith("file://")) { -+ } else if (path.startsWith(Constants.PATH_SEPARATOR)) { - return UnderFileSystemSingleLocal.getClient(); - } - CommonUtils.illegalArgumentException("Unknown under file system scheme " + path); - return null; - } - -- /** -- * Transform an input string like hdfs://host:port/dir, hdfs://host:port, file:///dir, /dir -- * into a pair of address and path. The returned pairs are ("hdfs://host:port", "/dir"), -- * ("hdfs://host:port", "/"), and ("/", "/dir"), respectively. -- * -- * @param path -- * the input path string -- * @return null if s does not start with tachyon://, tachyon-ft://, hdfs://, s3://, s3n://, -- * file://, /. Or a pair of strings denoting the under FS address and the relative path -- * relative to that address. For local FS (with prefixes file:// or /), the under FS -- * address is "/" and the path starts with "/". -- */ -- public static Pair parse(String path) { -- if (path == null) { -- return null; -- } else if (path.startsWith("tachyon://") || path.startsWith("tachyon-ft://") -- || path.startsWith("hdfs://") || path.startsWith("s3://") || path.startsWith("s3n://")) { -- String prefix = path.substring(0, path.indexOf("://") + 3); -- String body = path.substring(prefix.length()); -- if (body.contains(Constants.PATH_SEPARATOR)) { -- int ind = body.indexOf(Constants.PATH_SEPARATOR); -- return new Pair(prefix + body.substring(0, ind), body.substring(ind)); -- } else { -- return new Pair(path, Constants.PATH_SEPARATOR); -- } -- } else if (path.startsWith("file://") || path.startsWith(Constants.PATH_SEPARATOR)) { -- String prefix = "file://"; -- String suffix = path.startsWith(prefix) ? path.substring(prefix.length()) : path; -- return new Pair(Constants.PATH_SEPARATOR, suffix); -- } -- -- return null; -- } -- - public abstract void close() throws IOException; - - public abstract OutputStream create(String path) throws IOException; -@@ -115,25 +82,11 @@ public abstract class UnderFileSystem { - public abstract boolean isFile(String path) throws IOException; - - /** -- * Returns an array of strings naming the files and directories in the directory denoted by this -- * abstract pathname. -- * -- *

-- * If this abstract pathname does not denote a directory, then this method returns {@code null}. -- * Otherwise an array of strings is returned, one for each file or directory in the directory. -- * Names denoting the directory itself and the directory's parent directory are not included in -- * the result. Each string is a file name rather than a complete path. -- * -- *

-- * There is no guarantee that the name strings in the resulting array will appear in any specific -- * order; they are not, in particular, guaranteed to appear in alphabetical order. -+ * List all the files in the folder. - * - * @param path - * the path to list. -- * @return An array of strings naming the files and directories in the directory denoted by this -- * abstract pathname. The array will be empty if the directory is empty. Returns -- * {@code null} if this abstract pathname does not denote a directory, or if an I/O error -- * occurs. -+ * @return all the file names under the path. - * @throws IOException - */ - public abstract String[] list(String path) throws IOException; -diff --git a/main/src/main/java/tachyon/UnderFileSystemHdfs.java b/main/src/main/java/tachyon/UnderFileSystemHdfs.java -index 7766b2e..235fa2f 100644 ---- a/main/src/main/java/tachyon/UnderFileSystemHdfs.java -+++ b/main/src/main/java/tachyon/UnderFileSystemHdfs.java -@@ -31,6 +31,8 @@ import org.apache.hadoop.fs.permission.FsPermission; - import org.apache.hadoop.hdfs.DistributedFileSystem; - import org.apache.log4j.Logger; - -+import org.apache.hadoop.fs.glusterfs.*; -+ - import tachyon.conf.CommonConf; - import tachyon.util.CommonUtils; - -@@ -56,19 +58,28 @@ public class UnderFileSystemHdfs extends UnderFileSystem { - mUfsPrefix = fsDefaultName; - Configuration tConf = new Configuration(); - tConf.set("fs.defaultFS", fsDefaultName); -- tConf.set("fs.hdfs.impl", CommonConf.get().UNDERFS_HDFS_IMPL); -+ String glusterfsPrefix = "glusterfs:///"; -+ if (fsDefaultName.startsWith(glusterfsPrefix)) { -+ tConf.set("fs.glusterfs.impl", CommonConf.get().UNDERFS_GLUSTERFS_IMPL); -+ tConf.set("mapred.system.dir", CommonConf.get().UNDERFS_GLUSTERFS_MR_DIR); -+ tConf.set("fs.glusterfs.volumes", CommonConf.get().UNDERFS_GLUSTERFS_VOLUMES); -+ tConf.set("fs.glusterfs.volume.fuse." + CommonConf.get().UNDERFS_GLUSTERFS_VOLUMES, -+ CommonConf.get().UNDERFS_GLUSTERFS_MOUNTS); -+ }else{ -+ tConf.set("fs.hdfs.impl", CommonConf.get().UNDERFS_HDFS_IMPL); - -- // To disable the instance cache for hdfs client, otherwise it causes the -- // FileSystem closed exception. Being configurable for unit/integration -- // test only, and not expose to the end-user currently. -- tConf.set("fs.hdfs.impl.disable.cache", -- System.getProperty("fs.hdfs.impl.disable.cache", "false")); -+ // To disable the instance cache for hdfs client, otherwise it causes the -+ // FileSystem closed exception. Being configurable for unit/integration -+ // test only, and not expose to the end-user currently. -+ tConf.set("fs.hdfs.impl.disable.cache", -+ System.getProperty("fs.hdfs.impl.disable.cache", "false")); -+ } - - if (System.getProperty("fs.s3n.awsAccessKeyId") != null) { -- tConf.set("fs.s3n.awsAccessKeyId", System.getProperty("fs.s3n.awsAccessKeyId")); -+ tConf.set("fs.s3n.awsAccessKeyId", System.getProperty("fs.s3n.awsAccessKeyId")); - } - if (System.getProperty("fs.s3n.awsSecretAccessKey") != null) { -- tConf.set("fs.s3n.awsSecretAccessKey", System.getProperty("fs.s3n.awsSecretAccessKey")); -+ tConf.set("fs.s3n.awsSecretAccessKey", System.getProperty("fs.s3n.awsSecretAccessKey")); - } - Path path = new Path(fsDefaultName); - mFs = path.getFileSystem(tConf); -@@ -256,8 +267,7 @@ public class UnderFileSystemHdfs extends UnderFileSystem { - String[] rtn = new String[files.length]; - int i = 0; - for (FileStatus status : files) { -- // only return the relative path, to keep consistent with java.io.File.list() -- rtn[i ++] = status.getPath().toString().substring(path.length()); // mUfsPrefix -+ rtn[i ++] = status.getPath().toString().substring(mUfsPrefix.length()); - } - return rtn; - } else { -diff --git a/main/src/main/java/tachyon/UnderFileSystemSingleLocal.java b/main/src/main/java/tachyon/UnderFileSystemSingleLocal.java -index 250a939..76e6b67 100644 ---- a/main/src/main/java/tachyon/UnderFileSystemSingleLocal.java -+++ b/main/src/main/java/tachyon/UnderFileSystemSingleLocal.java -@@ -70,7 +70,7 @@ public class UnderFileSystemSingleLocal extends UnderFileSystem { - if (recursive && file.isDirectory()) { - String[] files = file.list(); - for (String child : files) { -- success = success && delete(CommonUtils.concat(path, child), true); -+ success = success && delete(path + Constants.PATH_SEPARATOR + child, true); - } - } - -diff --git a/main/src/main/java/tachyon/Users.java b/main/src/main/java/tachyon/Users.java -index 5ab1426..0eab6b4 100644 ---- a/main/src/main/java/tachyon/Users.java -+++ b/main/src/main/java/tachyon/Users.java -@@ -77,11 +77,11 @@ public class Users { - } - - public String getUserTempFolder(long userId) { -- return CommonUtils.concat(USER_FOLDER, userId); -+ return USER_FOLDER + Constants.PATH_SEPARATOR + userId; - } - - public String getUserUnderfsTempFolder(long userId) { -- return CommonUtils.concat(USER_UNDERFS_FOLDER, userId); -+ return USER_UNDERFS_FOLDER + Constants.PATH_SEPARATOR + userId; - } - - /** -diff --git a/main/src/main/java/tachyon/client/BlockOutStream.java b/main/src/main/java/tachyon/client/BlockOutStream.java -index 355771e..c19ac2b 100644 ---- a/main/src/main/java/tachyon/client/BlockOutStream.java -+++ b/main/src/main/java/tachyon/client/BlockOutStream.java -@@ -80,7 +80,7 @@ public class BlockOutStream extends OutStream { - throw new IOException(msg); - } - -- mLocalFilePath = CommonUtils.concat(localFolder.getPath(), BLOCK_ID); -+ mLocalFilePath = localFolder.getPath() + Constants.PATH_SEPARATOR + BLOCK_ID; - mLocalFile = new RandomAccessFile(mLocalFilePath, "rw"); - mLocalFileChannel = mLocalFile.getChannel(); - // change the permission of the temporary file in order that the worker can move it. -diff --git a/main/src/main/java/tachyon/client/FileOutStream.java b/main/src/main/java/tachyon/client/FileOutStream.java -index 7395d18..5f76584 100644 ---- a/main/src/main/java/tachyon/client/FileOutStream.java -+++ b/main/src/main/java/tachyon/client/FileOutStream.java -@@ -24,7 +24,6 @@ import org.apache.log4j.Logger; - - import tachyon.Constants; - import tachyon.UnderFileSystem; --import tachyon.util.CommonUtils; - - /** - * FileOutStream implementation of TachyonFile. It can only be gotten by -@@ -61,7 +60,7 @@ public class FileOutStream extends OutStream { - mCachedBytes = 0; - - if (WRITE_TYPE.isThrough()) { -- mUnderFsFile = CommonUtils.concat(TFS.createAndGetUserUnderfsTempFolder(), FILE.FID); -+ mUnderFsFile = TFS.createAndGetUserUnderfsTempFolder() + Constants.PATH_SEPARATOR + FILE.FID; - UnderFileSystem underfsClient = UnderFileSystem.get(mUnderFsFile); - if (BLOCK_CAPACITY > Integer.MAX_VALUE) { - throw new IOException("BLOCK_CAPCAITY (" + BLOCK_CAPACITY + ") can not bigger than " -@@ -133,10 +132,9 @@ public class FileOutStream extends OutStream { - - @Override - public void flush() throws IOException { -- // TODO We only flush the checkpoint output stream. Flush for RAMFS block streams. -- if (mCheckpointOutputStream != null) { -- mCheckpointOutputStream.flush(); -- } -+ // We only flush the checkpoint output stream. -+ // TODO flushing for RAMFS block streams. -+ mCheckpointOutputStream.flush(); - } - - private void getNextBlock() throws IOException { -diff --git a/main/src/main/java/tachyon/client/RemoteBlockInStream.java b/main/src/main/java/tachyon/client/RemoteBlockInStream.java -index 3f8a5e5..11c85f9 100644 ---- a/main/src/main/java/tachyon/client/RemoteBlockInStream.java -+++ b/main/src/main/java/tachyon/client/RemoteBlockInStream.java -@@ -30,7 +30,6 @@ import tachyon.conf.UserConf; - import tachyon.thrift.ClientBlockInfo; - import tachyon.thrift.NetAddress; - import tachyon.worker.DataServerMessage; --import tachyon.util.CommonUtils; - - /** - * BlockInStream for remote block. -@@ -200,7 +199,8 @@ public class RemoteBlockInStream extends BlockInStream { - } - if (host.equals(InetAddress.getLocalHost().getHostName()) - || host.equals(InetAddress.getLocalHost().getHostAddress())) { -- String localFileName = CommonUtils.concat(TFS.getRootFolder(), blockInfo.blockId); -+ String localFileName = -+ TFS.getRootFolder() + Constants.PATH_SEPARATOR + blockInfo.blockId; - LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!"); - } - LOG.info(host + ":" + (port + 1) + " current host is " -diff --git a/main/src/main/java/tachyon/client/TachyonFS.java b/main/src/main/java/tachyon/client/TachyonFS.java -index bc520b9..c89ed82 100644 ---- a/main/src/main/java/tachyon/client/TachyonFS.java -+++ b/main/src/main/java/tachyon/client/TachyonFS.java -@@ -47,7 +47,6 @@ import tachyon.thrift.ClientFileInfo; - import tachyon.thrift.ClientRawTableInfo; - import tachyon.thrift.ClientWorkerInfo; - import tachyon.thrift.FileDoesNotExistException; --import tachyon.thrift.InvalidPathException; - import tachyon.thrift.NetAddress; - import tachyon.thrift.NoWorkerException; - import tachyon.thrift.TachyonException; -@@ -131,16 +130,6 @@ public class TachyonFS { - mAvailableSpaceBytes = 0L; - } - -- /** -- * Print out the string representation of this Tachyon server address. -- * -- * @return the string representation like tachyon://host:port or tachyon-ft://host:port -- */ -- @Override -- public String toString() { -- return (mZookeeperMode ? Constants.HEADER_FT : Constants.HEADER) + mMasterAddress.toString(); -- } -- - public synchronized void accessLocalBlock(long blockId) { - connect(); - if (mWorkerClient != null && mIsWorkerLocal) { -@@ -201,21 +190,6 @@ public class TachyonFS { - } - } - -- /** -- * Cleans the given path, throwing an IOException rather than an InvalidPathException. -- * -- * @param path -- * The path to clean -- * @return the cleaned path -- */ -- private synchronized String cleanPathIOException(String path) throws IOException { -- try { -- return CommonUtils.cleanPath(path); -- } catch (InvalidPathException e) { -- throw new IOException(e.getMessage()); -- } -- } -- - public synchronized void close() throws TException { - if (mMasterClient != null) { - mMasterClient.cleanConnect(); -@@ -384,7 +358,7 @@ public class TachyonFS { - if (!mConnected) { - return -1; - } -- path = cleanPathIOException(path); -+ path = CommonUtils.cleanPath(path); - int fid = -1; - try { - fid = mMasterClient.user_createFile(path, blockSizeByte); -@@ -400,7 +374,7 @@ public class TachyonFS { - if (!mConnected) { - return -1; - } -- path = cleanPathIOException(path); -+ path = CommonUtils.cleanPath(path); - int fid = -1; - try { - fid = mMasterClient.user_createFileOnCheckpoint(path, checkpointPath); -@@ -421,7 +395,8 @@ public class TachyonFS { - if (!mConnected) { - return -1; - } -- path = cleanPathIOException(path); -+ path = CommonUtils.cleanPath(path); -+ - if (columns < 1 || columns > CommonConf.get().MAX_COLUMNS) { - throw new IOException("Column count " + columns + " is smaller than 1 or " + "bigger than " - + CommonConf.get().MAX_COLUMNS); -@@ -599,7 +574,7 @@ public class TachyonFS { - return null; - } - ClientFileInfo ret; -- path = cleanPathIOException(path); -+ path = CommonUtils.cleanPath(path); - if (useCachedMetadata && mCachedClientFileInfos.containsKey(path)) { - return mCachedClientFileInfos.get(path); - } -@@ -660,7 +635,7 @@ public class TachyonFS { - - public synchronized TachyonFile getFile(String path, boolean useCachedMetadata) - throws IOException { -- path = cleanPathIOException(path); -+ path = CommonUtils.cleanPath(path); - ClientFileInfo clientFileInfo = getClientFileInfo(path, useCachedMetadata); - if (clientFileInfo == null) { - return null; -@@ -724,7 +699,7 @@ public class TachyonFS { - return -1; - } - int fid = -1; -- path = cleanPathIOException(path); -+ path = CommonUtils.cleanPath(path); - try { - fid = mMasterClient.user_getFileId(path); - } catch (TException e) { -@@ -796,7 +771,7 @@ public class TachyonFS { - String getLocalFilename(long blockId) { - String rootFolder = getRootFolder(); - if (rootFolder != null) { -- String localFileName = CommonUtils.concat(rootFolder, blockId); -+ String localFileName = rootFolder + Constants.PATH_SEPARATOR + blockId; - File file = new File(localFileName); - if (file.exists()) { - return localFileName; -@@ -855,7 +830,7 @@ public class TachyonFS { - - public synchronized RawTable getRawTable(String path) throws IOException { - connect(); -- path = cleanPathIOException(path); -+ path = CommonUtils.cleanPath(path); - ClientRawTableInfo clientRawTableInfo; - try { - clientRawTableInfo = mMasterClient.user_getClientRawTableInfoByPath(path); -@@ -1007,7 +982,7 @@ public class TachyonFS { - if (!mConnected) { - return false; - } -- path = cleanPathIOException(path); -+ path = CommonUtils.cleanPath(path); - try { - return mMasterClient.user_mkdir(path); - } catch (TException e) { -diff --git a/main/src/main/java/tachyon/client/TachyonFile.java b/main/src/main/java/tachyon/client/TachyonFile.java -index eb09173..aab2f1f 100644 ---- a/main/src/main/java/tachyon/client/TachyonFile.java -+++ b/main/src/main/java/tachyon/client/TachyonFile.java -@@ -31,7 +31,6 @@ import tachyon.conf.UserConf; - import tachyon.thrift.ClientBlockInfo; - import tachyon.thrift.NetAddress; - import tachyon.worker.DataServerMessage; --import tachyon.util.CommonUtils; - - /** - * Tachyon File. -@@ -108,8 +107,8 @@ public class TachyonFile implements Comparable { - * is no guarantee that the file still exists after this call returns, as Tachyon may evict blocks - * from memory at any time. - * -- * @param blockIndex -- * The index of the block in the file. -+ * @param blockId -+ * The id of the block. - * @return filename on local file system or null if file not present on local file system. - * @throws IOException - */ -@@ -121,9 +120,8 @@ public class TachyonFile implements Comparable { - - public List getLocationHosts() throws IOException { - List locations = TFS.getClientBlockInfo(FID, 0).getLocations(); -- List ret = null; -+ List ret = new ArrayList(locations.size()); - if (locations != null) { -- ret = new ArrayList(locations.size()); - for (int k = 0; k < locations.size(); k ++) { - ret.add(locations.get(k).mHost); - } -@@ -235,7 +233,7 @@ public class TachyonFile implements Comparable { - } - if (host.equals(InetAddress.getLocalHost().getHostName()) - || host.equals(InetAddress.getLocalHost().getHostAddress())) { -- String localFileName = CommonUtils.concat(TFS.getRootFolder(), FID); -+ String localFileName = TFS.getRootFolder() + Constants.PATH_SEPARATOR + FID; - LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!"); - } else { - LOG.info(host + ":" + (port + 1) + " current host is " -diff --git a/main/src/main/java/tachyon/client/table/RawColumn.java b/main/src/main/java/tachyon/client/table/RawColumn.java -index f336888..9ad11ce 100644 ---- a/main/src/main/java/tachyon/client/table/RawColumn.java -+++ b/main/src/main/java/tachyon/client/table/RawColumn.java -@@ -20,7 +20,6 @@ import tachyon.Constants; - import tachyon.client.TachyonFS; - import tachyon.client.TachyonFile; - import tachyon.master.MasterInfo; --import tachyon.util.CommonUtils; - - /** - * The column of a RawTable. -@@ -38,8 +37,8 @@ public class RawColumn { - - // TODO creating file here should be based on id. - public boolean createPartition(int pId) throws IOException { -- return TFS.createFile(CommonUtils.concat(RAW_TABLE.getPath(), MasterInfo.COL + COLUMN_INDEX, -- pId)) > 0; -+ return TFS.createFile(RAW_TABLE.getPath() + Constants.PATH_SEPARATOR + MasterInfo.COL -+ + COLUMN_INDEX + Constants.PATH_SEPARATOR + pId) > 0; - } - - // TODO creating file here should be based on id. -@@ -49,14 +48,13 @@ public class RawColumn { - - // TODO creating file here should be based on id. - public TachyonFile getPartition(int pId, boolean cachedMetadata) throws IOException { -- return TFS.getFile( -- CommonUtils.concat(RAW_TABLE.getPath(), MasterInfo.COL + COLUMN_INDEX, pId), -- cachedMetadata); -+ return TFS.getFile(RAW_TABLE.getPath() + Constants.PATH_SEPARATOR + MasterInfo.COL -+ + COLUMN_INDEX + Constants.PATH_SEPARATOR + pId, cachedMetadata); - } - - // TODO creating file here should be based on id. - public int partitions() throws IOException { -- return TFS.getNumberOfFiles(CommonUtils.concat(RAW_TABLE.getPath(), MasterInfo.COL -- + COLUMN_INDEX)); -+ return TFS.getNumberOfFiles(RAW_TABLE.getPath() + Constants.PATH_SEPARATOR + MasterInfo.COL -+ + COLUMN_INDEX); - } --} -+} -\ No newline at end of file -diff --git a/main/src/main/java/tachyon/conf/CommonConf.java b/main/src/main/java/tachyon/conf/CommonConf.java -index 9215d99..5e8edfc 100644 ---- a/main/src/main/java/tachyon/conf/CommonConf.java -+++ b/main/src/main/java/tachyon/conf/CommonConf.java -@@ -49,6 +49,10 @@ public class CommonConf extends Utils { - public final String UNDERFS_WORKERS_FOLDER; - - public final String UNDERFS_HDFS_IMPL; -+ public final String UNDERFS_GLUSTERFS_IMPL; -+ public final String UNDERFS_GLUSTERFS_VOLUMES; -+ public final String UNDERFS_GLUSTERFS_MOUNTS; -+ public final String UNDERFS_GLUSTERFS_MR_DIR; - public final String WEB_RESOURCES; - public final boolean USE_ZOOKEEPER; - public final String ZOOKEEPER_ADDRESS; -@@ -79,7 +83,14 @@ public class CommonConf extends Utils { - getProperty("tachyon.workers.folder", UNDERFS_ADDRESS + "/tachyon/workers"); - UNDERFS_HDFS_IMPL = - getProperty("tachyon.underfs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); -- -+ UNDERFS_GLUSTERFS_IMPL = -+ getProperty("tachyon.underfs.glusterfs.impl", "org.apache.hadoop.fs.glusterfs.GlusterFileSystem"); -+ UNDERFS_GLUSTERFS_VOLUMES = -+ getProperty("tachyon.underfs.glusterfs.volumes", null); -+ UNDERFS_GLUSTERFS_MOUNTS = -+ getProperty("tachyon.underfs.glusterfs.mounts", null); -+ UNDERFS_GLUSTERFS_MR_DIR = -+ getProperty("tachyon.underfs.glusterfs.mapred.system.dir", "glusterfs:///mapred/system"); - USE_ZOOKEEPER = getBooleanProperty("tachyon.usezookeeper", false); - if (USE_ZOOKEEPER) { - ZOOKEEPER_ADDRESS = getProperty("tachyon.zookeeper.address"); -diff --git a/main/src/main/java/tachyon/conf/MasterConf.java b/main/src/main/java/tachyon/conf/MasterConf.java -index d045199..e0dc7d9 100644 ---- a/main/src/main/java/tachyon/conf/MasterConf.java -+++ b/main/src/main/java/tachyon/conf/MasterConf.java -@@ -44,7 +44,6 @@ public class MasterConf extends Utils { - public final String FORMAT_FILE_PREFIX; - public final String HOSTNAME; - public final int PORT; -- public final String MASTER_ADDRESS; - - public final int WEB_PORT; - public final String TEMPORARY_FOLDER; -@@ -70,9 +69,6 @@ public class MasterConf extends Utils { - - HOSTNAME = getProperty("tachyon.master.hostname", "localhost"); - PORT = getIntProperty("tachyon.master.port", Constants.DEFAULT_MASTER_PORT); -- MASTER_ADDRESS = -- (CommonConf.get().USE_ZOOKEEPER ? Constants.HEADER_FT : Constants.HEADER) + HOSTNAME + ":" -- + PORT; - WEB_PORT = getIntProperty("tachyon.master.web.port", Constants.DEFAULT_MASTER_WEB_PORT); - TEMPORARY_FOLDER = getProperty("tachyon.master.temporary.folder", "/tmp"); - -diff --git a/main/src/main/java/tachyon/hadoop/TFS.java b/main/src/main/java/tachyon/hadoop/TFS.java -index 2058fb6..87661e7 100644 ---- a/main/src/main/java/tachyon/hadoop/TFS.java -+++ b/main/src/main/java/tachyon/hadoop/TFS.java -@@ -177,8 +177,7 @@ public class TFS extends FileSystem { - Path hdfsPath = Utils.getHDFSPath(path); - FileSystem fs = hdfsPath.getFileSystem(getConf()); - if (fs.exists(hdfsPath)) { -- String ufsAddrPath = CommonUtils.concat(UNDERFS_ADDRESS, path); -- UnderfsUtil.loadUnderFs(mTFS, Constants.PATH_SEPARATOR, ufsAddrPath, new PrefixList(null)); -+ UnderfsUtil.getInfo(mTFS, UNDERFS_ADDRESS, path, new PrefixList(null)); - } - } - } -diff --git a/main/src/main/java/tachyon/hadoop/Utils.java b/main/src/main/java/tachyon/hadoop/Utils.java -index 9108117..2328a51 100644 ---- a/main/src/main/java/tachyon/hadoop/Utils.java -+++ b/main/src/main/java/tachyon/hadoop/Utils.java -@@ -23,7 +23,6 @@ import org.apache.hadoop.mapred.InputSplit; - import org.apache.log4j.Logger; - - import tachyon.Constants; --import tachyon.util.CommonUtils; - - public final class Utils { - private static final Logger LOG = Logger.getLogger(Constants.LOGGER_TYPE); -@@ -47,7 +46,7 @@ public final class Utils { - if (ret.equals("")) { - ret = path.getName(); - } else { -- ret = CommonUtils.concat(path.getName(), ret); -+ ret = path.getName() + Constants.PATH_SEPARATOR + ret; - } - path = path.getParent(); - } -diff --git a/main/src/main/java/tachyon/master/Dependency.java b/main/src/main/java/tachyon/master/Dependency.java -index 97abc8c..6a4f7e0 100644 ---- a/main/src/main/java/tachyon/master/Dependency.java -+++ b/main/src/main/java/tachyon/master/Dependency.java -@@ -13,7 +13,6 @@ import java.util.Set; - import org.apache.log4j.Logger; - - import tachyon.Constants; --import tachyon.conf.MasterConf; - import tachyon.io.Utils; - import tachyon.thrift.ClientDependencyInfo; - import tachyon.util.CommonUtils; -@@ -125,7 +124,6 @@ public class Dependency implements ImageWriter { - // TODO We should support different types of command in the future. - // For now, assume there is only one command model. - StringBuilder sb = new StringBuilder(parseCommandPrefix()); -- sb.append(" ").append(MasterConf.get().MASTER_ADDRESS); - sb.append(" ").append(ID); - for (int k = 0; k < CHILDREN_FILES.size(); k ++) { - int id = CHILDREN_FILES.get(k); -diff --git a/main/src/main/java/tachyon/master/EditLog.java b/main/src/main/java/tachyon/master/EditLog.java -index a13a30e..102ae8d 100644 ---- a/main/src/main/java/tachyon/master/EditLog.java -+++ b/main/src/main/java/tachyon/master/EditLog.java -@@ -58,11 +58,10 @@ public class EditLog { - - public static void deleteCompletedLogs(String path, int upTo) { - UnderFileSystem ufs = UnderFileSystem.get(path); -- String folder = -- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "completed"; -+ String folder = path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; - try { - for (int i = 0; i < upTo; i ++) { -- String toDelete = CommonUtils.concat(folder, i + ".editLog"); -+ String toDelete = folder + Constants.PATH_SEPARATOR + i + ".editLog"; - LOG.info("Deleting editlog " + toDelete); - ufs.delete(toDelete, true); - } -@@ -102,18 +101,20 @@ public class EditLog { - mBackUpLogStartNum = currentLogFileNum; - int numFiles = 1; - String completedPath = -- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "completed"; -+ path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; - if (!ufs.exists(completedPath)) { - LOG.info("No completed edit logs to be parsed"); - } else { -- while (ufs.exists(CommonUtils.concat(completedPath, (completedLogs ++) + ".editLog"))) { -+ while (ufs -+ .exists(completedPath + Constants.PATH_SEPARATOR + (completedLogs ++) + ".editLog")) { - numFiles ++; - } - } - String editLogs[] = new String[numFiles]; - for (int i = 0; i < numFiles; i ++) { - if (i != numFiles - 1) { -- editLogs[i] = CommonUtils.concat(completedPath, (i + currentLogFileNum) + ".editLog"); -+ editLogs[i] = -+ completedPath + Constants.PATH_SEPARATOR + (i + currentLogFileNum) + ".editLog"; - } else { - editLogs[i] = path; - } -@@ -214,16 +215,15 @@ public class EditLog { - - public static void markUpToDate(String path) { - UnderFileSystem ufs = UnderFileSystem.get(path); -- String folder = -- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "completed"; -+ String folder = path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; - try { - // delete all loaded editlogs since mBackupLogStartNum. -- String toDelete = CommonUtils.concat(folder, mBackUpLogStartNum + ".editLog"); -+ String toDelete = folder + Constants.PATH_SEPARATOR + mBackUpLogStartNum + ".editLog"; - while (ufs.exists(toDelete)) { - LOG.info("Deleting editlog " + toDelete); - ufs.delete(toDelete, true); - mBackUpLogStartNum ++; -- toDelete = CommonUtils.concat(folder, mBackUpLogStartNum + ".editLog"); -+ toDelete = folder + Constants.PATH_SEPARATOR + mBackUpLogStartNum + ".editLog"; - } - } catch (IOException e) { - CommonUtils.runtimeException(e); -@@ -253,24 +253,24 @@ public class EditLog { - UFS = UnderFileSystem.get(path); - if (mBackUpLogStartNum != -1) { - String folder = -- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "/completed"; -+ path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; - LOG.info("Deleting completed editlogs that are part of the image."); - deleteCompletedLogs(path, mBackUpLogStartNum); - LOG.info("Backing up logs from " + mBackUpLogStartNum + " since image is not updated."); - UFS.mkdirs(folder, true); -- String toRename = CommonUtils.concat(folder, mBackUpLogStartNum + ".editLog"); -+ String toRename = folder + Constants.PATH_SEPARATOR + mBackUpLogStartNum + ".editLog"; - int currentLogFileNum = 0; - while (UFS.exists(toRename)) { -- LOG.info("Rename " + toRename + " to " -- + CommonUtils.concat(folder, currentLogFileNum + ".editLog")); -+ LOG.info("Rename " + toRename + " to " + folder + Constants.PATH_SEPARATOR -+ + currentLogFileNum + ".editLog"); - currentLogFileNum ++; - mBackUpLogStartNum ++; -- toRename = CommonUtils.concat(folder, mBackUpLogStartNum + ".editLog"); -+ toRename = folder + Constants.PATH_SEPARATOR + mBackUpLogStartNum + ".editLog"; - } - if (UFS.exists(path)) { -- UFS.rename(path, CommonUtils.concat(folder, currentLogFileNum + ".editLog")); -- LOG.info("Rename " + path + " to " -- + CommonUtils.concat(folder, currentLogFileNum + ".editLog")); -+ UFS.rename(path, folder + Constants.PATH_SEPARATOR + currentLogFileNum + ".editLog"); -+ LOG.info("Rename " + path + " to " + folder + Constants.PATH_SEPARATOR -+ + currentLogFileNum + ".editLog"); - currentLogFileNum ++; - } - mBackUpLogStartNum = -1; -@@ -500,13 +500,14 @@ public class EditLog { - _close(); - LOG.info("Edit log max size reached, rotating edit log"); - String pathPrefix = -- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "completed"; -+ path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; - LOG.info("path: " + path + " prefix: " + pathPrefix); - try { - if (!UFS.exists(pathPrefix)) { - UFS.mkdirs(pathPrefix, true); - } -- String newPath = CommonUtils.concat(pathPrefix, (mCurrentLogFileNum ++) + ".editLog"); -+ String newPath = -+ pathPrefix + Constants.PATH_SEPARATOR + (mCurrentLogFileNum ++) + ".editLog"; - UFS.rename(path, newPath); - LOG.info("Renamed " + path + " to " + newPath); - OS = UFS.create(path); -diff --git a/main/src/main/java/tachyon/master/Inode.java b/main/src/main/java/tachyon/master/Inode.java -index e144f33..1e04cf4 100644 ---- a/main/src/main/java/tachyon/master/Inode.java -+++ b/main/src/main/java/tachyon/master/Inode.java -@@ -108,10 +108,10 @@ public abstract class Inode implements Comparable, ImageWriter { - - @Override - public synchronized String toString() { -- StringBuilder sb = new StringBuilder("Inode("); -+ StringBuilder sb = new StringBuilder("INode("); - sb.append("ID:").append(mId).append(", NAME:").append(mName); - sb.append(", PARENT_ID:").append(mParentId); - sb.append(", CREATION_TIME_MS:").append(CREATION_TIME_MS).append(")"); - return sb.toString(); - } --} -+} -\ No newline at end of file -diff --git a/main/src/main/java/tachyon/master/InodeFolder.java b/main/src/main/java/tachyon/master/InodeFolder.java -index 16fc58f..db3d853 100644 ---- a/main/src/main/java/tachyon/master/InodeFolder.java -+++ b/main/src/main/java/tachyon/master/InodeFolder.java -@@ -20,8 +20,8 @@ import java.io.IOException; - import java.util.ArrayList; - import java.util.HashSet; - import java.util.List; -+import java.util.Map; - import java.util.Set; --import java.util.Collections; - - import tachyon.io.Utils; - import tachyon.thrift.ClientFileInfo; -@@ -45,14 +45,9 @@ public class InodeFolder extends Inode { - int parentId = is.readInt(); - - int numberOfChildren = is.readInt(); -- Inode[] children = new Inode[numberOfChildren]; -+ int[] children = new int[numberOfChildren]; - for (int k = 0; k < numberOfChildren; k ++) { -- byte type = is.readByte(); -- if (type == Image.T_INODE_FILE) { -- children[k] = InodeFile.loadImage(is); -- } else { -- children[k] = InodeFolder.loadImage(is); -- } -+ children[k] = is.readInt(); - } - - InodeFolder folder = new InodeFolder(fileName, fileId, parentId, creationTimeMs); -@@ -60,41 +55,22 @@ public class InodeFolder extends Inode { - return folder; - } - -- private Set mChildren = new HashSet(); -+ private Set mChildren = new HashSet(); - - public InodeFolder(String name, int id, int parentId, long creationTimeMs) { - super(name, id, parentId, true, creationTimeMs); - } - -- /** -- * Adds the given inode to the set of children. -- * -- * @param child -- * The inode to add -- */ -- public synchronized void addChild(Inode child) { -- mChildren.add(child); -+ public synchronized void addChild(int childId) { -+ mChildren.add(childId); - } - -- /** -- * Adds the given inodes to the set of children. -- * -- * @param children -- * The inodes to add -- */ -- public synchronized void addChildren(Inode[] children) { -- for (Inode child : children) { -- addChild(child); -+ public synchronized void addChildren(int[] childrenIds) { -+ for (int k = 0; k < childrenIds.length; k ++) { -+ addChild(childrenIds[k]); - } - } - -- /** -- * Generates client file info for the folder. -- * -- * @param path -- * The path of the folder in the filesystem -- * @return the generated ClientFileInfo -- */ - @Override - public ClientFileInfo generateClientFileInfo(String path) { - ClientFileInfo ret = new ClientFileInfo(); -@@ -117,91 +93,37 @@ public class InodeFolder extends Inode { - return ret; - } - -- /** -- * Returns the child with the given id. -- * -- * @param fid -- * The id of the child -- * @return the inode with the given id, or null if there is no child with that id -- */ -- public synchronized Inode getChild(int fid) { -- for (Inode child : mChildren) { -- if (child.getId() == fid) { -- return child; -- } -- } -- return null; -- } -- -- /** -- * Returns the child with the given name. -- * -- * @param name -- * The name of the child -- * @return the inode with the given name, or null if there is no child with that name -- */ -- public synchronized Inode getChild(String name) { -- for (Inode child : mChildren) { -- if (child.getName().equals(name)) { -- return child; -+ public synchronized Inode getChild(String name, Map allInodes) { -+ Inode tInode = null; -+ for (int childId : mChildren) { -+ tInode = allInodes.get(childId); -+ if (tInode != null && tInode.getName().equals(name)) { -+ return tInode; - } - } - return null; - } - -- /** -- * Returns the folder's children. -- * -- * @return an unmodifiable set of the children inodes. -- */ -- public synchronized Set getChildren() { -- return Collections.unmodifiableSet(mChildren); -- } -- -- /** -- * Returns the ids of the children. -- * -- * @return the ids of the children -- */ - public synchronized List getChildrenIds() { - List ret = new ArrayList(mChildren.size()); -- for (Inode child : mChildren) { -- ret.add(child.getId()); -- } -+ ret.addAll(mChildren); - return ret; - } - -- /** -- * Returns the number of children the folder has. -- * -- * @return the number of children in the folder. -- */ - public synchronized int getNumberOfChildren() { - return mChildren.size(); - } - -- /** -- * Removes the given inode from the folder. -- * -- * @param child -- * The Inode to remove -- * @return true if the inode was removed, false otherwise. -- */ -- public synchronized boolean removeChild(Inode child) { -- return mChildren.remove(child); -+ public synchronized void removeChild(int id) { -+ mChildren.remove(id); - } - -- /** -- * Removes the given child from the folder. -- * -- * @param name -- * The name of the Inode to remove. -- * @return true if the inode was removed, false otherwise. -- */ -- public synchronized boolean removeChild(String name) { -- for (Inode child : mChildren) { -- if (child.getName().equals(name)) { -- mChildren.remove(child); -+ public synchronized boolean removeChild(String name, Map allInodes) { -+ Inode tInode = null; -+ for (int childId : mChildren) { -+ tInode = allInodes.get(childId); -+ if (tInode != null && tInode.getName().equals(name)) { -+ mChildren.remove(childId); - return true; - } - } -@@ -215,12 +137,6 @@ public class InodeFolder extends Inode { - return sb.toString(); - } - -- /** -- * Write an image of the folder. -- * -- * @param os -- * The output stream to write the folder to -- */ - @Override - public void writeImage(DataOutputStream os) throws IOException { - os.writeByte(Image.T_INODE_FOLDER); -@@ -232,8 +148,8 @@ public class InodeFolder extends Inode { - - List children = getChildrenIds(); - os.writeInt(children.size()); -- for (Inode inode : getChildren()) { -- inode.writeImage(os); -+ for (int k = 0; k < children.size(); k ++) { -+ os.writeInt(children.get(k)); - } - } --} -+} -\ No newline at end of file -diff --git a/main/src/main/java/tachyon/master/MasterInfo.java b/main/src/main/java/tachyon/master/MasterInfo.java -index af885e6..44675ad 100644 ---- a/main/src/main/java/tachyon/master/MasterInfo.java -+++ b/main/src/main/java/tachyon/master/MasterInfo.java -@@ -122,8 +122,7 @@ public class MasterInfo implements ImageWriter { - dep.addLostFile(tFile.getId()); - LOG.info("File " + tFile.getId() + " got lost from worker " + worker.getId() - + " . Trying to recompute it using dependency " + dep.ID); -- String path = getPath(tFile); -- if (path != null && !path.startsWith(MASTER_CONF.TEMPORARY_FOLDER)) { -+ if (!getPath(tFile).startsWith(MASTER_CONF.TEMPORARY_FOLDER)) { - mMustRecomputeDependencies.add(depId); - } - } -@@ -239,8 +238,8 @@ public class MasterInfo implements ImageWriter { - private Map mDependencies = new HashMap(); - private RawTables mRawTables = new RawTables(); - -- // TODO add initialization part for master failover or restart. All operations on these members -- // are synchronized on mDependencies. -+ // TODO add initialization part for master failover or restart. -+ // All operations on these members are synchronized on mDependencies. - private Set mUncheckpointedDependencies = new HashSet(); - private Set mPriorityDependencies = new HashSet(); - private Set mLostFiles = new HashSet(); -@@ -344,21 +343,11 @@ public class MasterInfo implements ImageWriter { - * Internal API. - * - * @param recursive -- * If recursive is true and the filesystem tree is not filled in all the way to path yet, -- * it fills in the missing components. - * @param path -- * The path to create - * @param directory -- * If true, creates an InodeFolder instead of an Inode - * @param blockSizeByte -- * If it's a file, the block size for the Inode - * @param creationTimeMs -- * The time the file was created -- * @param id -- * If not -1, use this id as the id to the inode we create at the given path. -- * Any intermediate directories created will use mInodeCounter, so using this parameter -- * assumes no intermediate directories will be created. -- * @return the id of the inode created at the given path -+ * @return - * @throws FileAlreadyExistException - * @throws InvalidPathException - * @throws BlockInfoException -@@ -378,9 +367,6 @@ public class MasterInfo implements ImageWriter { - synchronized (mRoot) { - Inode inode = getInode(pathNames); - if (inode != null) { -- if (inode.isDirectory() && directory) { -- return inode.getId(); -- } - LOG.info("FileAlreadyExistException: File " + path + " already exist."); - throw new FileAlreadyExistException("Path " + path + " already exist."); - } -@@ -435,7 +421,7 @@ public class MasterInfo implements ImageWriter { - } - - mInodes.put(ret.getId(), ret); -- ((InodeFolder) inode).addChild(ret); -+ ((InodeFolder) inode).addChild(ret.getId()); - - LOG.debug("createFile: File Created: " + ret + " parent: " + inode); - return ret.getId(); -@@ -473,7 +459,7 @@ public class MasterInfo implements ImageWriter { - } - - InodeFolder parent = (InodeFolder) mInodes.get(inode.getParentId()); -- parent.removeChild(inode); -+ parent.removeChild(inode.getId()); - mInodes.remove(inode.getId()); - if (inode.isFile()) { - String checkpointPath = ((InodeFile) inode).getCheckpointPath(); -@@ -614,43 +600,6 @@ public class MasterInfo implements ImageWriter { - } - - /** -- * Adds the given inode to mFileIdPinList, traversing folders recursively. Used while loading an -- * image. -- * -- * @param inode -- * The inode to add -- */ -- private void addToFileIdPinList(Inode inode) throws IOException { -- if (inode.isFile() && ((InodeFile) inode).isPin()) { -- synchronized (mFileIdPinList) { -- mFileIdPinList.add(inode.getId()); -- } -- } else if (inode.isDirectory()) { -- for (Inode child : ((InodeFolder) inode).getChildren()) { -- addToFileIdPinList(child); -- } -- } -- } -- -- /** -- * While loading an image, addToInodeMap will map the various ids to their inodes. -- * -- * @param inode -- * The inode to add -- * @param map -- * The map to add the inodes to -- */ -- private void addToInodeMap(Inode inode, Map map) { -- map.put(inode.getId(), inode); -- if (inode.isDirectory()) { -- InodeFolder inodeFolder = (InodeFolder) inode; -- for (Inode child : inodeFolder.getChildren()) { -- addToInodeMap(child, map); -- } -- } -- } -- -- /** - * A worker cache a block in its memory. - * - * @param workerId -@@ -827,7 +776,7 @@ public class MasterInfo implements ImageWriter { - } - - for (int k = 0; k < columns; k ++) { -- mkdir(CommonUtils.concat(path, COL + k)); -+ mkdir(path + Constants.PATH_SEPARATOR + COL + k); - } - - return id; -@@ -1116,6 +1065,23 @@ public class MasterInfo implements ImageWriter { - } - - /** -+ * Get the path of a file with the given id -+ * -+ * @param fileId -+ * The id of the file to look up -+ * @return the path of the file -+ */ -+ public String getFileNameById(int fileId) throws FileDoesNotExistException { -+ synchronized (mRoot) { -+ Inode inode = mInodes.get(fileId); -+ if (inode == null) { -+ throw new FileDoesNotExistException("FileId " + fileId + " does not exist"); -+ } -+ return getPath(inode); -+ } -+ } -+ -+ /** - * Get the file id's of the given paths. It recursively scans directories for the file id's inside - * of them. - * -@@ -1153,11 +1119,14 @@ public class MasterInfo implements ImageWriter { - } - - if (inode.isDirectory()) { -- Set children = ((InodeFolder) inode).getChildren(); -+ List childernIds = ((InodeFolder) inode).getChildrenIds(); - -+ if (!path.endsWith(Constants.PATH_SEPARATOR)) { -+ path += Constants.PATH_SEPARATOR; -+ } - synchronized (mRoot) { -- for (Inode child : children) { -- ret.add(getClientFileInfo(child.getId())); -+ for (int k : childernIds) { -+ ret.add(getClientFileInfo(k)); - } - } - } else { -@@ -1183,9 +1152,10 @@ public class MasterInfo implements ImageWriter { - InodeFolder tFolder = tPair.getFirst(); - String curPath = tPair.getSecond(); - -- Set children = tFolder.getChildren(); -- for (Inode tInode : children) { -- String newPath = CommonUtils.concat(curPath, tInode.getName()); -+ List childrenIds = tFolder.getChildrenIds(); -+ for (int id : childrenIds) { -+ Inode tInode = mInodes.get(id); -+ String newPath = curPath + Constants.PATH_SEPARATOR + tInode.getName(); - if (tInode.isDirectory()) { - nodesQueue.add(new Pair((InodeFolder) tInode, newPath)); - } else if (((InodeFile) tInode).isFullyInMemory()) { -@@ -1236,7 +1206,7 @@ public class MasterInfo implements ImageWriter { - if (cur.isFile()) { - return null; - } -- cur = ((InodeFolder) cur).getChild(name); -+ cur = ((InodeFolder) cur).getChild(name, mInodes); - } - - return cur; -@@ -1304,24 +1274,8 @@ public class MasterInfo implements ImageWriter { - if (inode.getParentId() == 1) { - return Constants.PATH_SEPARATOR + inode.getName(); - } -- return CommonUtils.concat(getPath(mInodes.get(inode.getParentId())), inode.getName()); -- } -- } -- -- /** -- * Get the path of a file with the given id -- * -- * @param fileId -- * The id of the file to look up -- * @return the path of the file -- */ -- public String getPath(int fileId) throws FileDoesNotExistException { -- synchronized (mRoot) { -- Inode inode = mInodes.get(fileId); -- if (inode == null) { -- throw new FileDoesNotExistException("FileId " + fileId + " does not exist"); -- } -- return getPath(inode); -+ return getPath(mInodes.get(inode.getParentId())) + Constants.PATH_SEPARATOR -+ + inode.getName(); - } - } - -@@ -1475,7 +1429,7 @@ public class MasterInfo implements ImageWriter { - * If true, select a random worker - * @param host - * If random is false, select a worker on this host -- * @return the address of the selected worker, or null if no address could be found -+ * @return the address of the selected worker - */ - public NetAddress getWorker(boolean random, String host) { - synchronized (mWorkers) { -@@ -1593,15 +1547,17 @@ public class MasterInfo implements ImageWriter { - if (inode.isFile()) { - ret.add(inode.getId()); - } else if (recursive) { -- Queue queue = new LinkedList(); -- queue.addAll(((InodeFolder) inode).getChildren()); -+ Queue queue = new LinkedList(); -+ queue.addAll(((InodeFolder) inode).getChildrenIds()); - - while (!queue.isEmpty()) { -- Inode qinode = queue.poll(); -- if (qinode.isDirectory()) { -- queue.addAll(((InodeFolder) inode).getChildren()); -+ int id = queue.poll(); -+ inode = mInodes.get(id); -+ -+ if (inode.isDirectory()) { -+ queue.addAll(((InodeFolder) inode).getChildrenIds()); - } else { -- ret.add(qinode.getId()); -+ ret.add(id); - } - } - } -@@ -1649,16 +1605,15 @@ public class MasterInfo implements ImageWriter { - inode = InodeFolder.loadImage(is); - } - -+ LOG.info("Putting " + inode); - if (inode.getId() > mInodeCounter.get()) { - mInodeCounter.set(inode.getId()); - } - -- addToInodeMap(inode, mInodes); -- addToFileIdPinList(inode); -- - if (inode.getId() == 1) { - mRoot = (InodeFolder) inode; - } -+ mInodes.put(inode.getId(), inode); - } else if (Image.T_RAW_TABLE == type) { - mRawTables.loadImage(is); - } else { -@@ -1689,15 +1644,22 @@ public class MasterInfo implements ImageWriter { - if (inode.isFile()) { - ret.add(path); - } else { -- Set children = ((InodeFolder) inode).getChildren(); -+ List childrenIds = ((InodeFolder) inode).getChildrenIds(); -+ -+ if (!path.endsWith(Constants.PATH_SEPARATOR)) { -+ path += Constants.PATH_SEPARATOR; -+ } - ret.add(path); - - synchronized (mRoot) { -- for (Inode child : children) { -- if (recursive) { -- ret.addAll(ls(CommonUtils.concat(path, child.getName()), true)); -- } else { -- ret.add(CommonUtils.concat(path, child.getName())); -+ for (int k : childrenIds) { -+ inode = mInodes.get(k); -+ if (inode != null) { -+ if (recursive) { -+ ret.addAll(ls(path + inode.getName(), true)); -+ } else { -+ ret.add(path + inode.getName()); -+ } - } - } - } -@@ -1711,7 +1673,7 @@ public class MasterInfo implements ImageWriter { - * - * @param path - * The path to create a directory at -- * @return true if and only if the directory was created; false otherwise -+ * @return true if the creation was successful and false if it wasn't - */ - public boolean mkdir(String path) throws FileAlreadyExistException, InvalidPathException, - TachyonException { -@@ -1836,9 +1798,9 @@ public class MasterInfo implements ImageWriter { - - srcInode.setName(dstName); - InodeFolder parent = (InodeFolder) mInodes.get(srcInode.getParentId()); -- parent.removeChild(srcInode); -+ parent.removeChild(srcInode.getId()); - srcInode.setParentId(dstFolderInode.getId()); -- ((InodeFolder) dstFolderInode).addChild(srcInode); -+ ((InodeFolder) dstFolderInode).addChild(srcInode.getId()); - - mJournal.getEditLog().rename(srcInode.getId(), dstPath); - mJournal.getEditLog().flush(); -@@ -2055,13 +2017,30 @@ public class MasterInfo implements ImageWriter { - * The output stream to write the image to - */ - public void writeImage(DataOutputStream os) throws IOException { -+ Queue folderQueue = new LinkedList(); -+ - synchronized (mRoot) { -- synchronized (mDependencies) { -- for (Dependency dep : mDependencies.values()) { -- dep.writeImage(os); -- } -+ for (Dependency dep : mDependencies.values()) { -+ dep.writeImage(os); - } -+ - mRoot.writeImage(os); -+ folderQueue.add(mRoot); -+ while (!folderQueue.isEmpty()) { -+ List childrenIds = folderQueue.poll().getChildrenIds(); -+ for (int id : childrenIds) { -+ Inode tInode = mInodes.get(id); -+ tInode.writeImage(os); -+ if (tInode.isDirectory()) { -+ folderQueue.add((InodeFolder) tInode); -+ } else if (((InodeFile) tInode).isPin()) { -+ synchronized (mFileIdPinList) { -+ mFileIdPinList.add(tInode.getId()); -+ } -+ } -+ } -+ } -+ - mRawTables.writeImage(os); - - os.writeByte(Image.T_CHECKPOINT); -diff --git a/main/src/main/java/tachyon/util/CommonUtils.java b/main/src/main/java/tachyon/util/CommonUtils.java -index cd43ccf..4b326c6 100644 ---- a/main/src/main/java/tachyon/util/CommonUtils.java -+++ b/main/src/main/java/tachyon/util/CommonUtils.java -@@ -19,7 +19,6 @@ import java.io.IOException; - import java.io.InputStream; - import java.io.OutputStream; - import java.io.PrintStream; --import java.math.BigDecimal; - import java.net.InetSocketAddress; - import java.nio.ByteBuffer; - import java.text.DateFormat; -@@ -29,7 +28,6 @@ import java.util.Date; - import java.util.List; - import java.util.Scanner; - --import org.apache.commons.io.FilenameUtils; - import org.apache.log4j.Logger; - - import tachyon.Constants; -@@ -81,17 +79,14 @@ public final class CommonUtils { - changeLocalFilePermission(filePath, "777"); - } - -- /** -- * Checks and normalizes the given path -- * -- * @param path -- * The path to clean up -- * @return a normalized version of the path, with single separators between path components and -- * dot components resolved -- */ -- public static String cleanPath(String path) throws InvalidPathException { -- validatePath(path); -- return FilenameUtils.separatorsToUnix(FilenameUtils.normalizeNoEndSeparator(path)); -+ public static String cleanPath(String path) throws IOException { -+ if (path == null || path.isEmpty()) { -+ throw new IOException("Path (" + path + ") is invalid."); -+ } -+ while (path.endsWith(Constants.PATH_SEPARATOR) && path.length() > 1) { -+ path = path.substring(0, path.length() - 1); -+ } -+ return path; - } - - public static ByteBuffer cloneByteBuffer(ByteBuffer buf) { -@@ -109,31 +104,6 @@ public final class CommonUtils { - return ret; - } - -- /** -- * Add the path component to the base path -- * -- * @param args -- * The components to concatenate -- * @return the concatenated path -- */ -- public static String concat(Object... args) { -- if (args.length == 0) { -- return ""; -- } -- String retPath = args[0].toString(); -- for (int k = 1; k < args.length; k ++) { -- while (retPath.endsWith(Constants.PATH_SEPARATOR)) { -- retPath = retPath.substring(0, retPath.length() - 1); -- } -- if (args[k].toString().startsWith(Constants.PATH_SEPARATOR)) { -- retPath += args[k].toString(); -- } else { -- retPath += Constants.PATH_SEPARATOR + args[k].toString(); -- } -- } -- return retPath; -- } -- - public static String convertByteArrayToStringWithoutEscape(byte[] data) { - StringBuilder sb = new StringBuilder(data.length); - for (int i = 0; i < data.length; i ++) { -@@ -214,7 +184,8 @@ public final class CommonUtils { - * @return the name of the file - */ - public static String getName(String path) throws InvalidPathException { -- return FilenameUtils.getName(cleanPath(path)); -+ String[] pathNames = getPathComponents(path); -+ return pathNames[pathNames.length - 1]; - } - - /** -@@ -225,8 +196,8 @@ public final class CommonUtils { - * @return the path split into components - */ - public static String[] getPathComponents(String path) throws InvalidPathException { -- path = cleanPath(path); -- if (isRoot(path)) { -+ validatePath(path); -+ if (path.length() == 1 && path.equals(Constants.PATH_SEPARATOR)) { - String[] ret = new String[1]; - ret[0] = ""; - return ret; -@@ -260,17 +231,6 @@ public final class CommonUtils { - throw new IllegalArgumentException(msg); - } - -- /** -- * Check if the given path is the root. -- * -- * @param path -- * The path to check -- * @return true if the path is the root -- */ -- public static boolean isRoot(String path) throws InvalidPathException { -- return Constants.PATH_SEPARATOR.equals(cleanPath(path)); -- } -- - public static String listToString(List list) { - StringBuilder sb = new StringBuilder(); - for (int k = 0; k < list.size(); k ++) { -@@ -340,11 +300,6 @@ public final class CommonUtils { - return (long) (ret * Constants.GB + alpha); - } else if (end.equals("tb")) { - return (long) (ret * Constants.TB + alpha); -- } else if (end.equals("pb")) { -- // When parsing petabyte values, we can't multiply with doubles and longs, since that will -- // lose presicion with such high numbers. Therefore we use a BigDecimal. -- BigDecimal PBDecimal = new BigDecimal(Constants.PB); -- return PBDecimal.multiply(BigDecimal.valueOf(ret)).longValue(); - } else { - runtimeException("Fail to parse " + ori + " as memory size"); - return -1; -@@ -437,8 +392,8 @@ public final class CommonUtils { - } - - public static void validatePath(String path) throws InvalidPathException { -- if (path == null || path.isEmpty() || !path.startsWith(Constants.PATH_SEPARATOR) -- || path.contains(" ")) { -+ if (path == null || !path.startsWith(Constants.PATH_SEPARATOR) -+ || (path.length() > 1 && path.endsWith(Constants.PATH_SEPARATOR)) || path.contains(" ")) { - throw new InvalidPathException("Path " + path + " is invalid."); - } - } -diff --git a/main/src/main/java/tachyon/util/UnderfsUtil.java b/main/src/main/java/tachyon/util/UnderfsUtil.java -index bd8bfdd..75b25fc 100644 ---- a/main/src/main/java/tachyon/util/UnderfsUtil.java -+++ b/main/src/main/java/tachyon/util/UnderfsUtil.java -@@ -18,14 +18,19 @@ import java.io.IOException; - import java.util.LinkedList; - import java.util.Queue; - -+import org.apache.hadoop.conf.Configuration; - import org.apache.log4j.Logger; -+import org.apache.thrift.TException; - - import tachyon.Constants; --import tachyon.Pair; - import tachyon.PrefixList; - import tachyon.UnderFileSystem; - import tachyon.Version; - import tachyon.client.TachyonFS; -+import tachyon.thrift.FileAlreadyExistException; -+import tachyon.thrift.FileDoesNotExistException; -+import tachyon.thrift.InvalidPathException; -+import tachyon.thrift.SuspectedFileSizeException; - - /** - * Utilities related to under filesystem -@@ -33,167 +38,73 @@ import tachyon.client.TachyonFS; - public class UnderfsUtil { - private static Logger LOG = Logger.getLogger(Constants.LOGGER_TYPE); - -- /** -- * Create a new path relative to a given TFS root. -- * -- * @param tfsRootPath -- * the destination point in TFS to load the under FS path onto -- * @param ufsRootPath -- * the source path in the under FS to be loaded -- * @param path -- * the path relative to ufsRootPath of a file to be loaded -- * @return the new path relative to tfsRootPath. -- */ -- private static String createTFSPath(String tfsRootPath, String ufsRootPath, String path) { -- String filePath = path.substring(ufsRootPath.length()); -- if (filePath.isEmpty()) { -- // retrieve the basename in ufsRootPath -- filePath = path.substring(ufsRootPath.lastIndexOf(Constants.PATH_SEPARATOR) + 1); -- } -- return CommonUtils.concat(tfsRootPath, filePath); -- } -- -- /** -- * Load files under path "ufsAddrRootPath" (excluding excludePathPrefix relative to the path) -- * to the given tfs under a given destination path. -- * -- * @param tfsAddrRootPath -- * the TFS address and path to load the src files, like "tachyon://host:port/dest". -- * @param ufsAddrRootPath -- * the address and root path of the under FS, like "hdfs://host:port/src". -- * @param excludePaths -- * paths to exclude from ufsRootPath, which will not be loaded in TFS. -- * @throws IOException -- */ -- public static void loadUnderFs(String tfsAddrRootPath, String ufsAddrRootPath, -- String excludePaths) throws IOException { -- Pair tfsPair = UnderFileSystem.parse(tfsAddrRootPath); -- String tfsAddress = tfsPair.getFirst(); -- String tfsRootPath = tfsPair.getSecond(); -- TachyonFS tfs = TachyonFS.get(tfsAddress); -- -- PrefixList excludePathPrefix = new PrefixList(excludePaths, ";"); -- -- loadUnderFs(tfs, tfsRootPath, ufsAddrRootPath, excludePathPrefix); -- } -- -- /** -- * Load files under path "ufsAddress/ufsRootPath" (excluding excludePathPrefix) -- * to the given tfs under the given tfsRootPath directory. -- * -- * @param tfs -- * the TFS handler created out of address like "tachyon://host:port" -- * @param tfsRootPath -- * the destination point in TFS to load the under FS path onto -- * @param ufsAddrRootPath -- * the address and root path of the under FS, like "hdfs://host:port/dir". -- * @param excludePathPrefix -- * paths to exclude from ufsRootPath, which will not be registered in TFS. -- * @throws IOException -- */ -- public static void loadUnderFs(TachyonFS tfs, String tfsRootPath, String ufsAddrRootPath, -+ public static void getInfo(TachyonFS tfs, String underfsAddress, String rootPath, - PrefixList excludePathPrefix) throws IOException { -- LOG.info(tfs + tfsRootPath + " " + ufsAddrRootPath + " " + excludePathPrefix); -- -- Pair tfsPair = UnderFileSystem.parse(ufsAddrRootPath); -- String ufsAddress = tfsPair.getFirst(); -- String ufsRootPath = tfsPair.getSecond(); -+ LOG.info(tfs + " " + underfsAddress + " " + rootPath + " " + excludePathPrefix); - -- if (!tfs.exist(tfsRootPath)) { -- tfs.mkdir(tfsRootPath); -- // TODO Add the following. -- // if (tfs.mkdir(tfsRootPath)) { -- // LOG.info("directory " + tfsRootPath + " does not exist in Tachyon: created"); -- // } else { -- // throw new IOException("Failed to create folder in Tachyon: " + tfsRootPath); -- // } -- } -- -- // create the under FS handler (e.g. hdfs, local FS, s3 etc.) -- UnderFileSystem ufs = UnderFileSystem.get(ufsAddress); -+ Configuration tConf = new Configuration(); -+ tConf.set("fs.default.name", underfsAddress + rootPath); -+ // TODO Use underfs to make this generic. -+ UnderFileSystem fs = UnderFileSystem.get(underfsAddress); - -- Queue ufsPathQueue = new LinkedList(); -- if (excludePathPrefix.outList(ufsRootPath)) { -- ufsPathQueue.add(ufsAddrRootPath); -+ Queue pathQueue = new LinkedList(); -+ if (excludePathPrefix.outList(rootPath)) { -+ pathQueue.add(underfsAddress + rootPath); - } -- -- while (!ufsPathQueue.isEmpty()) { -- String ufsPath = ufsPathQueue.poll(); // the absolute path -- if (ufs.isFile(ufsPath)) { -- String tfsPath = createTFSPath(tfsRootPath, ufsAddrRootPath, ufsPath); -- if (tfs.exist(tfsPath)) { -- LOG.info("File " + tfsPath + " already exists in Tachyon."); -+ while (!pathQueue.isEmpty()) { -+ String path = pathQueue.poll(); -+ if (fs.isFile(path)) { -+ String filePath = path.substring(underfsAddress.length()); -+ if (tfs.exist(filePath)) { -+ LOG.info("File " + filePath + " already exists in Tachyon."); - continue; - } -- int fileId = tfs.createFile(tfsPath, ufsPath); -+ int fileId = tfs.createFile(filePath, path); - if (fileId == -1) { -- LOG.info("Failed to create tachyon file: " + tfsPath); -+ LOG.info("Failed to create tachyon file: " + filePath); - } else { -- LOG.info("Create tachyon file " + tfsPath + " with file id " + fileId + " and " -- + "checkpoint location " + ufsPath); -+ LOG.info("Create tachyon file " + filePath + " with file id " + fileId + " and " -+ + "checkpoint location " + path); - } -- } else { // ufsPath is a directory -- String[] files = ufs.list(ufsPath); // ufs.list() returns relative path -+ } else { -+ String[] files = fs.list(path); - if (files != null) { - for (String filePath : files) { - LOG.info("Get: " + filePath); -- String aPath = CommonUtils.concat(ufsPath, filePath); -- String checkPath = aPath.substring(ufsAddrRootPath.length()); -- if (checkPath.startsWith(Constants.PATH_SEPARATOR)) { -- checkPath = checkPath.substring(Constants.PATH_SEPARATOR.length()); -- } -- if (excludePathPrefix.inList(checkPath)) { -- LOG.info("excluded: " + checkPath); -- } else { -- ufsPathQueue.add(aPath); -+ if (excludePathPrefix.outList(filePath)) { -+ pathQueue.add(underfsAddress + filePath); - } - } - } -- String tfsPath = createTFSPath(tfsRootPath, ufsAddrRootPath, ufsPath); -- if (!tfs.exist(tfsPath)) { -- tfs.mkdir(tfsPath); -- // TODO Add the following. -- // if (tfs.mkdir(tfsPath)) { -- // LOG.info("Created tachyon folder " + tfsPath + " with checkpoint location " + ufsPath); -- // } else { -- // LOG.info("Failed to create tachyon folder: " + tfsPath); -- // } -+ String filePath = path.substring(underfsAddress.length()); -+ if (!tfs.exist(filePath)) { -+ tfs.mkdir(filePath); - } - } - } - } - -- public static void main(String[] args) { -- if (!(args.length == 2 || args.length == 3)) { -- printUsage(); -+ public static void main(String[] args) throws SuspectedFileSizeException, InvalidPathException, -+ IOException, FileDoesNotExistException, FileAlreadyExistException, TException { -+ if (!(args.length == 3 || args.length == 4)) { -+ String prefix = -+ "java -cp target/tachyon-" + Version.VERSION + "-jar-with-dependencies.jar " -+ + "tachyon.util.UnderfsUtil "; -+ System.out.println("Usage: " + prefix + " " -+ + "[]"); -+ System.out.println("Example: " + prefix -+ + "tachyon://127.0.0.1:19998 hdfs://localhost:9000 / /tachyon"); - System.exit(-1); - } - -- String exList = (args.length == 3) ? args[2] : ""; -- -- try { -- loadUnderFs(args[0], args[1], exList); -- } catch (Exception e) { -- e.printStackTrace(); -- printUsage(); -- System.exit(-2); -+ PrefixList tExcludePathPrefix = null; -+ if (args.length == 4) { -+ tExcludePathPrefix = new PrefixList(args[3], ";"); -+ } else { -+ tExcludePathPrefix = new PrefixList(null); - } - -+ getInfo(TachyonFS.get(args[0]), args[1], args[2], tExcludePathPrefix); - System.exit(0); - } -- -- public static void printUsage() { -- String cmd = -- "java -cp target/tachyon-" + Version.VERSION + "-jar-with-dependencies.jar " -- + "tachyon.util.UnderfsUtil "; -- -- System.out.println("Usage: " + cmd + " " -- + "[]"); -- System.out -- .println("Example: " + cmd + "tachyon://127.0.0.1:19998/a hdfs://localhost:9000/b c"); -- System.out.println("Example: " + cmd + "tachyon://127.0.0.1:19998/a file:///b c"); -- System.out.println("Example: " + cmd + "tachyon://127.0.0.1:19998/a /b c"); -- System.out.print("In the TFS, all files under local FS /b will be registered under /a, "); -- System.out.println("except for those with prefix c"); -- } - } -diff --git a/main/src/main/java/tachyon/web/WebInterfaceBrowseServlet.java b/main/src/main/java/tachyon/web/WebInterfaceBrowseServlet.java -index 6b1917a..c0d0ebf 100644 ---- a/main/src/main/java/tachyon/web/WebInterfaceBrowseServlet.java -+++ b/main/src/main/java/tachyon/web/WebInterfaceBrowseServlet.java -@@ -363,8 +363,9 @@ public class WebInterfaceBrowseServlet extends HttpServlet { - String currentPath = Constants.PATH_SEPARATOR; - pathInfos[0] = new UiFileInfo(mMasterInfo.getClientFileInfo(currentPath)); - for (int i = 1; i < splitPath.length - 1; i ++) { -- currentPath = CommonUtils.concat(currentPath, splitPath[i]); -+ currentPath = currentPath + splitPath[i]; - pathInfos[i] = new UiFileInfo(mMasterInfo.getClientFileInfo(currentPath)); -+ currentPath = currentPath + Constants.PATH_SEPARATOR; - } - request.setAttribute("pathInfos", pathInfos); - return; -diff --git a/main/src/main/java/tachyon/web/WebInterfaceDependencyServlet.java b/main/src/main/java/tachyon/web/WebInterfaceDependencyServlet.java -index d7474bb..787e981 100644 ---- a/main/src/main/java/tachyon/web/WebInterfaceDependencyServlet.java -+++ b/main/src/main/java/tachyon/web/WebInterfaceDependencyServlet.java -@@ -35,10 +35,10 @@ public class WebInterfaceDependencyServlet extends HttpServlet { - try { - ClientDependencyInfo dependencyInfo = mMasterInfo.getClientDependencyInfo(dependencyId); - for (int pId : dependencyInfo.parents) { -- parentFileNames.add(mMasterInfo.getPath(pId)); -+ parentFileNames.add(mMasterInfo.getFileNameById(pId)); - } - for (int cId : dependencyInfo.children) { -- childrenFileNames.add(mMasterInfo.getPath(cId)); -+ childrenFileNames.add(mMasterInfo.getFileNameById(cId)); - } - } catch (DependencyDoesNotExistException ddnee) { - request.setAttribute("error", ddnee.getMessage()); -diff --git a/main/src/main/java/tachyon/worker/DataServerMessage.java b/main/src/main/java/tachyon/worker/DataServerMessage.java -index de01bb3..04b6a06 100644 ---- a/main/src/main/java/tachyon/worker/DataServerMessage.java -+++ b/main/src/main/java/tachyon/worker/DataServerMessage.java -@@ -77,7 +77,7 @@ public class DataServerMessage { - throw new IOException("Length can not be negative except -1: " + len); - } - -- String filePath = CommonUtils.concat(WorkerConf.get().DATA_FOLDER, blockId); -+ String filePath = WorkerConf.get().DATA_FOLDER + Constants.PATH_SEPARATOR + blockId; - ret.LOG.info("Try to response remote requst by reading from " + filePath); - RandomAccessFile file = new RandomAccessFile(filePath, "r"); - -diff --git a/main/src/main/java/tachyon/worker/WorkerStorage.java b/main/src/main/java/tachyon/worker/WorkerStorage.java -index d0e089f..a02fdf3 100644 ---- a/main/src/main/java/tachyon/worker/WorkerStorage.java -+++ b/main/src/main/java/tachyon/worker/WorkerStorage.java -@@ -174,8 +174,9 @@ public class WorkerStorage { - - // TODO checkpoint process. In future, move from midPath to dstPath should be done by - // master -- String midPath = CommonUtils.concat(mUnderfsWorkerDataFolder, fileId); -- String dstPath = CommonUtils.concat(CommonConf.get().UNDERFS_DATA_FOLDER, fileId); -+ String midPath = mUnderfsWorkerDataFolder + Constants.PATH_SEPARATOR + fileId; -+ String dstPath = -+ CommonConf.get().UNDERFS_DATA_FOLDER + Constants.PATH_SEPARATOR + fileId; - LOG.info("Thread " + ID + " is checkpointing file " + fileId + " from " - + mLocalDataFolder.toString() + " to " + midPath + " to " + dstPath); - -@@ -196,7 +197,8 @@ public class WorkerStorage { - long fileSizeByte = 0; - for (int k = 0; k < fileInfo.blockIds.size(); k ++) { - File tempFile = -- new File(CommonUtils.concat(mLocalDataFolder.toString(), fileInfo.blockIds.get(k))); -+ new File(mLocalDataFolder.toString() + Constants.PATH_SEPARATOR -+ + fileInfo.blockIds.get(k)); - fileSizeByte += tempFile.length(); - InputStream is = new FileInputStream(tempFile); - byte[] buf = new byte[16 * Constants.KB]; -@@ -313,7 +315,8 @@ public class WorkerStorage { - mLocalDataFolder = new File(dataFolder); - mLocalUserFolder = - new File(mLocalDataFolder.toString(), WorkerConf.get().USER_TEMP_RELATIVE_FOLDER); -- mUnderfsWorkerFolder = CommonUtils.concat(COMMON_CONF.UNDERFS_WORKERS_FOLDER, mWorkerId); -+ mUnderfsWorkerFolder = -+ COMMON_CONF.UNDERFS_WORKERS_FOLDER + Constants.PATH_SEPARATOR + mWorkerId; - mUnderfsWorkerDataFolder = mUnderfsWorkerFolder + "/data"; - mUnderFs = UnderFileSystem.get(COMMON_CONF.UNDERFS_ADDRESS); - mUsers = new Users(mLocalUserFolder.toString(), mUnderfsWorkerFolder); -@@ -359,8 +362,8 @@ public class WorkerStorage { - public void addCheckpoint(long userId, int fileId) throws FileDoesNotExistException, - SuspectedFileSizeException, FailedToCheckpointException, BlockInfoException, TException { - // TODO This part need to be changed. -- String srcPath = CommonUtils.concat(getUserUnderfsTempFolder(userId), fileId); -- String dstPath = CommonUtils.concat(COMMON_CONF.UNDERFS_DATA_FOLDER, fileId); -+ String srcPath = getUserUnderfsTempFolder(userId) + Constants.PATH_SEPARATOR + fileId; -+ String dstPath = COMMON_CONF.UNDERFS_DATA_FOLDER + Constants.PATH_SEPARATOR + fileId; - try { - if (!mUnderFs.rename(srcPath, dstPath)) { - throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath); -@@ -403,8 +406,8 @@ public class WorkerStorage { - - public void cacheBlock(long userId, long blockId) throws FileDoesNotExistException, - SuspectedFileSizeException, BlockInfoException, TException { -- File srcFile = new File(CommonUtils.concat(getUserTempFolder(userId), blockId)); -- File dstFile = new File(CommonUtils.concat(mLocalDataFolder, blockId)); -+ File srcFile = new File(getUserTempFolder(userId) + Constants.PATH_SEPARATOR + blockId); -+ File dstFile = new File(mLocalDataFolder + Constants.PATH_SEPARATOR + blockId); - long fileSizeBytes = srcFile.length(); - if (!srcFile.exists()) { - throw new FileDoesNotExistException("File " + srcFile + " does not exist."); -@@ -453,21 +456,21 @@ public class WorkerStorage { - * The block to be removed. - * @return Removed file size in bytes. - */ -- private long freeBlock(long blockId) { -+ private synchronized long freeBlock(long blockId) { - Long freedFileBytes = null; -- synchronized (mLatestBlockAccessTimeMs) { -- if (mBlockSizes.containsKey(blockId)) { -- mWorkerSpaceCounter.returnUsedBytes(mBlockSizes.get(blockId)); -- File srcFile = new File(CommonUtils.concat(mLocalDataFolder, blockId)); -- srcFile.delete(); -+ if (mBlockSizes.containsKey(blockId)) { -+ mWorkerSpaceCounter.returnUsedBytes(mBlockSizes.get(blockId)); -+ File srcFile = new File(mLocalDataFolder + Constants.PATH_SEPARATOR + blockId); -+ srcFile.delete(); -+ synchronized (mLatestBlockAccessTimeMs) { - mLatestBlockAccessTimeMs.remove(blockId); - freedFileBytes = mBlockSizes.remove(blockId); - mRemovedBlockList.add(blockId); - mMemoryData.remove(blockId); -- LOG.info("Removed Data " + blockId); -- } else { -- LOG.warn("File " + blockId + " does not exist in memory."); - } -+ LOG.info("Removed Data " + blockId); -+ } else { -+ LOG.warn("File " + blockId + " does not exist in memory."); - } - - return freedFileBytes == null ? 0 : freedFileBytes; -@@ -705,7 +708,7 @@ public class WorkerStorage { - RandomAccessFile localFile = new RandomAccessFile(file, "r"); - ByteBuffer buf = localFile.getChannel().map(MapMode.READ_ONLY, 0, file.length()); - -- String ufsOrphanBlock = CommonUtils.concat(mUnderfsOrphansFolder, blockId); -+ String ufsOrphanBlock = mUnderfsOrphansFolder + Constants.PATH_SEPARATOR + blockId; - OutputStream os = mUnderFs.create(ufsOrphanBlock); - int BULKSIZE = 1024 * 64; - byte[] bulk = new byte[BULKSIZE]; -diff --git a/main/src/main/webapp/browse.jsp b/main/src/main/webapp/browse.jsp -index d73a10e..5281511 100644 ---- a/main/src/main/webapp/browse.jsp -+++ b/main/src/main/webapp/browse.jsp -@@ -1,7 +1,5 @@ - <%@ page import="java.util.*" %> - <%@ page import="tachyon.web.*" %> --<%@ page import="static org.apache.commons.lang.StringEscapeUtils.escapeHtml" %> --<%@ page import="static java.net.URLEncoder.encode" %> - - - -@@ -35,11 +33,11 @@ -

- -@@ -115,7 +113,7 @@ - <% if (!fileInfo.getIsDirectory()) { %> - - <% } %> -- "><%= escapeHtml(fileInfo.getName()) %> -+ <%= fileInfo.getName() %> - - <%= fileInfo.getSize() %> - <%= fileInfo.getBlockSizeBytes() %> -diff --git a/main/src/main/webapp/viewFile.jsp b/main/src/main/webapp/viewFile.jsp -index 2eb564e..31ce58b 100644 ---- a/main/src/main/webapp/viewFile.jsp -+++ b/main/src/main/webapp/viewFile.jsp -@@ -1,7 +1,5 @@ - <%@ page import="java.util.*" %> - <%@ page import="tachyon.web.*" %> --<%@ page import="static org.apache.commons.lang.StringEscapeUtils.escapeHtml" %> --<%@ page import="static java.net.URLEncoder.encode" %> - - - -@@ -17,7 +15,7 @@ - function displayContent() - { - var tmp = document.getElementById("offset").value; -- window.location.href = "./browse?path=<%= encode(request.getAttribute("currentPath").toString(), "UTF-8") %>&offset=" + tmp; -+ window.location.href = "./browse?path=<%= request.getAttribute("currentPath") %>&offset=" + tmp; - } - -
-@@ -38,7 +36,7 @@ -

- <%= request.getAttribute("invalidPathError") %> -

--

<%= escapeHtml(request.getAttribute("currentPath").toString()) %>: First 5KB from <%= request.getAttribute("viewingOffset") %> in ASCII

-+

<%= request.getAttribute("currentPath") %>: First 5KB from <%= request.getAttribute("viewingOffset") %> in ASCII

- -
- -diff --git a/main/src/test/java/tachyon/PrefixListTest.java b/main/src/test/java/tachyon/PrefixListTest.java -index 3b31644..8d06b0d 100644 ---- a/main/src/test/java/tachyon/PrefixListTest.java -+++ b/main/src/test/java/tachyon/PrefixListTest.java -@@ -83,19 +83,4 @@ public final class PrefixListTest { - Assert.assertTrue(prefixList.outList("")); - Assert.assertTrue(prefixList.outList(null)); - } -- -- @Test -- public void toStringTest() { -- PrefixList prefixList = new PrefixList("", ";"); -- Assert.assertEquals(prefixList.toString(), ";"); -- -- prefixList = new PrefixList(";", ";"); -- Assert.assertEquals(prefixList.toString(), ""); -- -- prefixList = new PrefixList("a/b;c", ";"); -- Assert.assertEquals(prefixList.toString(), "a/b;c;"); -- -- prefixList = new PrefixList("a/b;c;", ";"); -- Assert.assertEquals(prefixList.toString(), "a/b;c;"); -- } - } -diff --git a/main/src/test/java/tachyon/UnderFileSystemCluster.java b/main/src/test/java/tachyon/UnderFileSystemCluster.java -index 2ca049e..387283e 100644 ---- a/main/src/test/java/tachyon/UnderFileSystemCluster.java -+++ b/main/src/test/java/tachyon/UnderFileSystemCluster.java -@@ -109,7 +109,7 @@ public abstract class UnderFileSystemCluster { - String path = getUnderFilesystemAddress() + Constants.PATH_SEPARATOR; - UnderFileSystem ufs = UnderFileSystem.get(path); - for (String p : ufs.list(path)) { -- ufs.delete(CommonUtils.concat(path, p), true); -+ ufs.delete(path + p, true); - } - } - } -diff --git a/main/src/test/java/tachyon/UnderFileSystemTest.java b/main/src/test/java/tachyon/UnderFileSystemTest.java -deleted file mode 100644 -index 087694e..0000000 ---- a/main/src/test/java/tachyon/UnderFileSystemTest.java -+++ /dev/null -@@ -1,67 +0,0 @@ --/* -- * Licensed to the Apache Software Foundation (ASF) under one or more -- * contributor license agreements. See the NOTICE file distributed with -- * this work for additional information regarding copyright ownership. -- * The ASF licenses this file to You under the Apache License, Version 2.0 -- * (the "License"); you may not use this file except in compliance with -- * the License. You may obtain a copy of the License at -- * http://www.apache.org/licenses/LICENSE-2.0 -- * Unless required by applicable law or agreed to in writing, software -- * distributed under the License is distributed on an "AS IS" BASIS, -- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -- * See the License for the specific language governing permissions and -- * limitations under the License. -- */ --package tachyon; -- --import org.junit.Assert; --import org.junit.Test; -- --/** -- * Unit tests for {@link UnderFileSystem} -- */ --public final class UnderFileSystemTest { -- -- @Test -- public void parseTest() { -- Pair result = UnderFileSystem.parse("/path"); -- Assert.assertEquals(result.getFirst(), "/"); -- Assert.assertEquals(result.getSecond(), "/path"); -- -- result = UnderFileSystem.parse("file:///path"); -- Assert.assertEquals(result.getFirst(), "/"); -- Assert.assertEquals(result.getSecond(), "/path"); -- -- result = UnderFileSystem.parse("tachyon://localhost:19998"); -- Assert.assertEquals(result.getFirst(), "tachyon://localhost:19998"); -- Assert.assertEquals(result.getSecond(), "/"); -- -- result = UnderFileSystem.parse("tachyon://localhost:19998/"); -- Assert.assertEquals(result.getFirst(), "tachyon://localhost:19998"); -- Assert.assertEquals(result.getSecond(), "/"); -- -- result = UnderFileSystem.parse("tachyon://localhost:19998/path"); -- Assert.assertEquals(result.getFirst(), "tachyon://localhost:19998"); -- Assert.assertEquals(result.getSecond(), "/path"); -- -- result = UnderFileSystem.parse("tachyon-ft://localhost:19998/path"); -- Assert.assertEquals(result.getFirst(), "tachyon-ft://localhost:19998"); -- Assert.assertEquals(result.getSecond(), "/path"); -- -- result = UnderFileSystem.parse("hdfs://localhost:19998/path"); -- Assert.assertEquals(result.getFirst(), "hdfs://localhost:19998"); -- Assert.assertEquals(result.getSecond(), "/path"); -- -- result = UnderFileSystem.parse("s3://localhost:19998/path"); -- Assert.assertEquals(result.getFirst(), "s3://localhost:19998"); -- Assert.assertEquals(result.getSecond(), "/path"); -- -- result = UnderFileSystem.parse("s3n://localhost:19998/path"); -- Assert.assertEquals(result.getFirst(), "s3n://localhost:19998"); -- Assert.assertEquals(result.getSecond(), "/path"); -- -- Assert.assertEquals(UnderFileSystem.parse(null), null); -- Assert.assertEquals(UnderFileSystem.parse(""), null); -- Assert.assertEquals(UnderFileSystem.parse("anythingElse"), null); -- } --} -diff --git a/main/src/test/java/tachyon/client/TachyonFSTest.java b/main/src/test/java/tachyon/client/TachyonFSTest.java -index ab54ff8..f99afb5 100644 ---- a/main/src/test/java/tachyon/client/TachyonFSTest.java -+++ b/main/src/test/java/tachyon/client/TachyonFSTest.java -@@ -338,13 +338,6 @@ public class TachyonFSTest { - } - - @Test -- public void toStringTest() throws Exception { -- String tfsAddress = "tachyon://localhost:19998"; -- TachyonFS tfs = TachyonFS.get(tfsAddress); -- Assert.assertEquals(tfs.toString(), "tachyon://localhost/127.0.0.1:19998"); -- } -- -- @Test - public void unlockBlockTest1() throws IOException { - TachyonFile tFile = null; - int numOfFiles = 5; -diff --git a/main/src/test/java/tachyon/command/TFsShellTest.java b/main/src/test/java/tachyon/command/TFsShellTest.java -index cb3a05a..a3b887e 100644 ---- a/main/src/test/java/tachyon/command/TFsShellTest.java -+++ b/main/src/test/java/tachyon/command/TFsShellTest.java -@@ -363,10 +363,10 @@ public class TFsShellTest { - Assert.assertTrue(tFile.isDirectory()); - } - -- @Test -+ @Test(expected = IOException.class) - public void mkdirExistingTest() throws IOException { -- Assert.assertEquals(0, mFsShell.mkdir(new String[] { "mkdir", "/testFile1" })); -- Assert.assertEquals(0, mFsShell.mkdir(new String[] { "mkdir", "/testFile1" })); -+ mFsShell.mkdir(new String[] { "mkdir", "/testFile1" }); -+ mFsShell.mkdir(new String[] { "mkdir", "/testFile1" }); - } - - @Test(expected = IOException.class) -diff --git a/main/src/test/java/tachyon/hadoop/GlusterFSTest.java b/main/src/test/java/tachyon/hadoop/GlusterFSTest.java -new file mode 100644 -index 0000000..2ef6a74 ---- /dev/null -+++ b/main/src/test/java/tachyon/hadoop/GlusterFSTest.java -@@ -0,0 +1,55 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+package tachyon.hadoop; -+ -+import java.io.IOException; -+ -+import org.apache.hadoop.fs.glusterfs.*; -+import org.junit.After; -+import org.junit.Before; -+import org.junit.Assert; -+import org.junit.Test; -+ -+import tachyon.UnderFileSystem; -+ -+/** -+ * Unit tests for tachyon.hadoop.GlusterFS. -+ */ -+public class GlusterFSTest { -+ private UnderFileSystem mHcfs = null; -+ -+ @After -+ public final void after() throws Exception { -+ System.clearProperty("fs.default.name"); -+ System.clearProperty("tachyon.underfs.glusterfs.mapred.system.dir"); -+ System.clearProperty("tachyon.underfs.glusterfs.mounts"); -+ System.clearProperty("tachyon.underfs.glusterfs.volumes"); -+ } -+ -+ @Before -+ public final void before() throws IOException { -+ System.setProperty("fs.default.name", "glusterfs:///"); -+ System.setProperty("tachyon.underfs.glusterfs.mapred.system.dir", "glusterfs:///mapred/system"); -+ System.setProperty("tachyon.underfs.glusterfs.mounts", "/vol"); -+ System.setProperty("tachyon.underfs.glusterfs.volumes", "tachyon_vol"); -+ } -+ @Test -+ public void createGlusterFS() throws Exception { -+ mHcfs = UnderFileSystem.get("glusterfs:///"); -+ -+ Assert.assertTrue(mHcfs.create("tachyon_test") != null); -+ Assert.assertTrue(mHcfs.delete("tachyon_test", false)); -+ } -+} -diff --git a/main/src/test/java/tachyon/hadoop/HadoopCompatibleFSTest.java b/main/src/test/java/tachyon/hadoop/HadoopCompatibleFSTest.java -new file mode 100644 -index 0000000..359ee97 ---- /dev/null -+++ b/main/src/test/java/tachyon/hadoop/HadoopCompatibleFSTest.java -@@ -0,0 +1,59 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one or more -+ * contributor license agreements. See the NOTICE file distributed with -+ * this work for additional information regarding copyright ownership. -+ * The ASF licenses this file to You under the Apache License, Version 2.0 -+ * (the "License"); you may not use this file except in compliance with -+ * the License. You may obtain a copy of the License at -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+package tachyon.hadoop; -+ -+import java.io.IOException; -+ -+// import org.apache.hadoop.conf.Configuration; -+// import org.apache.hadoop.fs.FileSystem; -+// import org.apache.hadoop.hdfs.DistributedFileSystem; -+// import org.apache.hadoop.hdfs.MiniDFSCluster; -+// import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; -+import org.junit.After; -+import org.junit.Before; -+import tachyon.client.TachyonFS; -+import tachyon.master.LocalTachyonCluster; -+ -+/** -+ * Unit tests for tachyon.hadoop.HadoopCompatibleFS. -+ */ -+public class HadoopCompatibleFSTest { -+ private LocalTachyonCluster mLocalTachyonCluster = null; -+ private TachyonFS mTfs = null; -+ -+ // private MiniDFSCluster mDfsCluster = null; -+ // private DistributedFileSystem mDfs = null; -+ -+ @After -+ public final void after() throws Exception { -+ mLocalTachyonCluster.stop(); -+ System.clearProperty("tachyon.user.quota.unit.bytes"); -+ } -+ -+ @Before -+ public final void before() throws IOException { -+ System.setProperty("tachyon.user.quota.unit.bytes", "1000"); -+ mLocalTachyonCluster = new LocalTachyonCluster(10000); -+ mLocalTachyonCluster.start(); -+ mTfs = mLocalTachyonCluster.getClient(); -+ -+ // Configuration conf = new Configuration(); -+ // System.setProperty("fs.default.name", "hdfs://localhost:54310"); -+ // mDfsCluster = new MiniDFSCluster(); -+ // FileSystem fs = mDfsCluster.getFileSystem(); -+ // Assert.assertTrue("Not a HDFS: "+fs.getUri(), fs instanceof DistributedFileSystem); -+ // mDfs = (DistributedFileSystem) fs; -+ } -+} -diff --git a/main/src/test/java/tachyon/master/InodeFolderTest.java b/main/src/test/java/tachyon/master/InodeFolderTest.java -index 3009032..4b5ea94 100644 ---- a/main/src/test/java/tachyon/master/InodeFolderTest.java -+++ b/main/src/test/java/tachyon/master/InodeFolderTest.java -@@ -31,10 +31,8 @@ public class InodeFolderTest { - @Test - public void addChildrenTest() { - InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); -- InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); -- InodeFile inodeFile2 = new InodeFile("testFile2", 3, 1, 1000, System.currentTimeMillis()); -- inodeFolder.addChild(inodeFile1); -- inodeFolder.addChild(inodeFile2); -+ inodeFolder.addChild(2); -+ inodeFolder.addChild(3); - Assert.assertEquals(2, (int) inodeFolder.getChildrenIds().get(0)); - Assert.assertEquals(3, (int) inodeFolder.getChildrenIds().get(1)); - } -@@ -44,12 +42,14 @@ public class InodeFolderTest { - InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); - InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); - InodeFile inodeFile2 = new InodeFile("testFile2", 3, 1, 1000, System.currentTimeMillis()); -- InodeFile inodeFile3 = new InodeFile("testFile3", 4, 1, 1000, System.currentTimeMillis()); -- inodeFolder.addChild(inodeFile1); -- inodeFolder.addChild(inodeFile2); -- inodeFolder.addChild(inodeFile3); -+ inodeFolder.addChild(2); -+ inodeFolder.addChild(3); -+ inodeFolder.addChild(4); -+ Map testMap = new HashMap(2); -+ testMap.put(2, inodeFile1); -+ testMap.put(3, inodeFile2); - Assert.assertEquals(3, inodeFolder.getNumberOfChildren()); -- inodeFolder.removeChild("testFile1"); -+ inodeFolder.removeChild("testFile1", testMap); - Assert.assertEquals(2, inodeFolder.getNumberOfChildren()); - Assert.assertFalse(inodeFolder.getChildrenIds().contains(2)); - } -@@ -90,21 +90,18 @@ public class InodeFolderTest { - @Test - public void removeChildTest() { - InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); -- InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); -- inodeFolder.addChild(inodeFile1); -+ inodeFolder.addChild(2); - Assert.assertEquals(1, inodeFolder.getNumberOfChildren()); -- inodeFolder.removeChild(inodeFile1); -+ inodeFolder.removeChild(2); - Assert.assertEquals(0, inodeFolder.getNumberOfChildren()); - } - - @Test - public void removeNonExistentChildTest() { - InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); -- InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); -- InodeFile inodeFile2 = new InodeFile("testFile2", 3, 1, 1000, System.currentTimeMillis()); -- inodeFolder.addChild(inodeFile1); -+ inodeFolder.addChild(2); - Assert.assertEquals(1, inodeFolder.getNumberOfChildren()); -- inodeFolder.removeChild(inodeFile2); -+ inodeFolder.removeChild(3); - Assert.assertEquals(1, inodeFolder.getNumberOfChildren()); - } - -@@ -118,9 +115,8 @@ public class InodeFolderTest { - @Test - public void sameIdChildrenTest() { - InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); -- InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); -- inodeFolder.addChild(inodeFile1); -- inodeFolder.addChild(inodeFile1); -+ inodeFolder.addChild(2); -+ inodeFolder.addChild(2); - Assert.assertTrue(inodeFolder.getChildrenIds().get(0) == 2); - Assert.assertEquals(1, inodeFolder.getNumberOfChildren()); - } -diff --git a/main/src/test/java/tachyon/util/CommonUtilsTest.java b/main/src/test/java/tachyon/util/CommonUtilsTest.java -index b8bb69b..b4154b2 100644 ---- a/main/src/test/java/tachyon/util/CommonUtilsTest.java -+++ b/main/src/test/java/tachyon/util/CommonUtilsTest.java -@@ -52,12 +52,5 @@ public class CommonUtilsTest { - Assert.assertEquals(k * Constants.TB / 10, CommonUtils.parseMemorySize(k / 10.0 + "TB")); - Assert.assertEquals(k * Constants.TB / 10, CommonUtils.parseMemorySize(k / 10.0 + "tB")); - } -- // We stop the pb test before 8192, since 8192 petabytes is beyond the scope of a java long. -- for (long k = 0; k < 8192; k ++) { -- Assert.assertEquals(k * Constants.PB / 10, CommonUtils.parseMemorySize(k / 10.0 + "pb")); -- Assert.assertEquals(k * Constants.PB / 10, CommonUtils.parseMemorySize(k / 10.0 + "Pb")); -- Assert.assertEquals(k * Constants.PB / 10, CommonUtils.parseMemorySize(k / 10.0 + "PB")); -- Assert.assertEquals(k * Constants.PB / 10, CommonUtils.parseMemorySize(k / 10.0 + "pB")); -- } - } - } -diff --git a/main/src/test/java/tachyon/util/UnderfsUtilTest.java b/main/src/test/java/tachyon/util/UnderfsUtilTest.java -index 7a71f3a..860a598 100644 ---- a/main/src/test/java/tachyon/util/UnderfsUtilTest.java -+++ b/main/src/test/java/tachyon/util/UnderfsUtilTest.java -@@ -31,6 +31,8 @@ import tachyon.UnderFileSystem; - import tachyon.UnderFileSystemCluster; - import tachyon.client.TachyonFS; - import tachyon.master.LocalTachyonCluster; -+import tachyon.util.UnderfsUtil; -+import tachyon.util.CommonUtils; - - /** - * To test the utilities related to under filesystem, including loadufs and etc. -@@ -62,7 +64,7 @@ public class UnderfsUtilTest { - } - - @Test -- public void loadUnderFsTest() throws IOException { -+ public void getInfoTest() throws IOException { - if (!UnderFileSystemCluster.isUFSHDFS()) { - return; - } -@@ -82,8 +84,8 @@ public class UnderfsUtilTest { - CommonUtils.touch(mUnderfsAddress + inclusion + "/1"); - } - -- UnderfsUtil.loadUnderFs(mTfs, Constants.PATH_SEPARATOR, mUnderfsAddress -- + Constants.PATH_SEPARATOR, new PrefixList(Arrays.asList(exclusions))); -+ UnderfsUtil.getInfo(mTfs, mUnderfsAddress, Constants.PATH_SEPARATOR, -+ new PrefixList(Arrays.asList(exclusions))); - - List paths = null; - for (String exclusion : exclusions) { -diff --git a/pom.xml b/pom.xml -index c74d6bb..767fbaf 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -70,6 +70,16 @@ - false - - -+ -+ glusterfs-hadoop-repo -+ http://rhbd.s3.amazonaws.com/maven/repositories/internal/ -+ -+ true -+ -+ -+ false -+ -+ - - -