From 8fdb4b990c5be8c469a1f384f24db29492918e39 Mon Sep 17 00:00:00 2001 From: Timothy St. Clair Date: May 06 2014 20:42:12 +0000 Subject: Update to latest but not yet functional due to glusterfs + hadoop update --- diff --git a/amplab-tachyon.spec b/amplab-tachyon.spec index 72c6ba3..b23c096 100644 --- a/amplab-tachyon.spec +++ b/amplab-tachyon.spec @@ -1,12 +1,12 @@ -%global commit 4b9c806bbce73db39e010a8bafd031c9b9749882 +%global commit 71028e3478eac70b20d76ad75ae4fe771b2515da %global shortcommit %(c=%{commit}; echo ${c:0:7}) %global shortname tachyon Name: amplab-%{shortname} # Given the naming conflicts with other packages, and eventually this will # switch to apache-tachyon should -Version: 0.4.1 -Release: 2.SNAPSHOT.%{shortcommit}%{?dist} +Version: 0.5.0 +Release: 1.SNAPSHOT.%{shortcommit}%{?dist} Summary: Reliable file sharing at memory speed across cluster frameworks License: ASL 2.0 URL: http://tachyon-project.org/ @@ -19,9 +19,7 @@ Source5: %{shortname}-env.sh Patch0: tachyon-0.4.0-SNAPSHOT-log4props.patch Patch1: tachyon-0.4.0-defaults.patch - -# Will likely go away https://github.com/amplab/tachyon/pull/128 -Patch2: tachyon-0.4.1-permissions.patch +Patch2: tachyon-0.5.0-gluster.patch BuildRequires: java-devel BuildRequires: mvn(commons-io:commons-io) @@ -41,6 +39,10 @@ BuildRequires: mvn(org.eclipse.jetty:jetty-servlet) BuildRequires: mvn(org.glassfish.web:javax.servlet.jsp) BuildRequires: mvn(org.slf4j:slf4j-api) BuildRequires: mvn(org.slf4j:slf4j-log4j12) +BuildRequires: mvn(org.slf4j:slf4j-api) +BuildRequires: mvn(org.powermock:powermock-module-junit4) +BuildRequires: mvn(org.powermock:powermock-api-mockito) +BuildRequires: mvn(org.apache.hadoop.fs.glusterfs:glusterfs) # Test deps BuildRequires: mvn(junit:junit) @@ -81,8 +83,8 @@ This package contains javadoc for %{name}. find -name '*.class' -print -delete find -name '*.jar' -print -delete -%patch0 -p1 -%patch1 -p1 +%patch0 -F2 -p1 +%patch1 -F2 -p1 %patch2 -p1 sed -i "s|hadoop-client|hadoop-mapreduce-client-core|" pom.xml @@ -90,10 +92,10 @@ sed -i "s|hadoop-client|hadoop-mapreduce-client-core|" p %pom_xpath_remove "pom:repositories" # Remove unnecessary plugin -%pom_remove_plugin :maven-assembly-plugin +# %pom_remove_plugin :maven-assembly-plugin # Fix unavailable jetty-jsp-2.1 -%pom_remove_dep org.eclipse.jetty:jetty-jsp +#%pom_remove_dep org.eclipse.jetty:jetty-jsp %pom_add_dep org.glassfish.web:javax.servlet.jsp::compile #make additions for hadoop2 @@ -139,7 +141,7 @@ mkdir -p -m0755 %{buildroot}%{_var}/lib/%{shortname}/journal ####################### mkdir -p -m0755 %{buildroot}/%{_datadir}/%{shortname}/web -cp -rf src/main/java/tachyon/web/resources %{buildroot}/%{_datadir}/%{shortname}/web +cp -rf main/src/main/webapp %{buildroot}/%{_datadir}/%{shortname}/web ####################### # NOTE: The following is plugging into hadoop without @@ -188,6 +190,9 @@ mkdir -p -m0755 %{buildroot}/%{_datadir}/hadoop/common/lib %systemd_postun_with_restart %{shortname}-slave.service %{shortname}-master.service %changelog +* Mon May 5 2014 Timothy St. Clair - 0.5.0-1.SNAPSHOT.71028e3 +- Update to 0.5.0 + * Mon Feb 24 2014 Timothy St. Clair - 0.4.1-2.SNAPSHOT.4b9c806 - Update due to cascading dependencies around java-headless diff --git a/sources b/sources index 0914e16..3a3febd 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -0e1c55d5a5bd5a768bcb1066e5f74120 tachyon-0.4.1-4b9c806.tar.gz +e5c4849a38908e408a30073b2f67bbaf tachyon-0.5.0-71028e3.tar.gz diff --git a/tachyon-0.4.0-defaults.patch b/tachyon-0.4.0-defaults.patch index 1ed43d6..7a21177 100644 --- a/tachyon-0.4.0-defaults.patch +++ b/tachyon-0.4.0-defaults.patch @@ -54,23 +54,3 @@ index 9122bcd..66cb2a2 100755 done exit 0 -diff --git a/src/main/java/tachyon/conf/CommonConf.java b/src/main/java/tachyon/conf/CommonConf.java -index aad1b53..7359cbb 100644 ---- a/src/main/java/tachyon/conf/CommonConf.java -+++ b/src/main/java/tachyon/conf/CommonConf.java -@@ -43,14 +43,7 @@ public class CommonConf extends Utils { - public final boolean ASYNC_ENABLED; - - private CommonConf() { -- if (System.getProperty("tachyon.home") == null) { -- LOG.warn("tachyon.home is not set. Using /mnt/tachyon_default_home as the default value."); -- File file = new File("/mnt/tachyon_default_home"); -- if (!file.exists()) { -- file.mkdirs(); -- } -- } -- TACHYON_HOME = getProperty("tachyon.home", "/mnt/tachyon_default_home"); -+ TACHYON_HOME = getProperty("tachyon.home", "/var/lib/tachyon"); - WEB_RESOURCES = getProperty("tachyon.web.resources", TACHYON_HOME + "/src/main/java/tachyon/web/resources"); - UNDERFS_ADDRESS = getProperty("tachyon.underfs.address", TACHYON_HOME + "/underfs"); - UNDERFS_DATA_FOLDER = getProperty("tachyon.data.folder", UNDERFS_ADDRESS + "/tachyon/data"); diff --git a/tachyon-0.4.1-permissions.patch b/tachyon-0.4.1-permissions.patch deleted file mode 100644 index 1bbb807..0000000 --- a/tachyon-0.4.1-permissions.patch +++ /dev/null @@ -1,39 +0,0 @@ -diff --git a/src/main/java/tachyon/worker/WorkerStorage.java b/src/main/java/tachyon/worker/WorkerStorage.java -index 592ddf8..95bf1ce 100644 ---- a/src/main/java/tachyon/worker/WorkerStorage.java -+++ b/src/main/java/tachyon/worker/WorkerStorage.java -@@ -25,6 +25,9 @@ import java.io.RandomAccessFile; - import java.net.InetSocketAddress; - import java.nio.ByteBuffer; - import java.nio.channels.FileChannel.MapMode; -+import java.nio.file.Files; -+import java.nio.file.attribute.PosixFilePermission; -+import java.nio.file.attribute.PosixFilePermissions; - import java.util.ArrayList; - import java.util.HashMap; - import java.util.HashSet; -@@ -513,10 +516,16 @@ public class WorkerStorage { - private void initializeWorkerStorage() throws IOException, FileDoesNotExistException, - SuspectedFileSizeException, BlockInfoException, TException { - LOG.info("Initializing the worker storage."); -+ -+ // Set Default directory permissions to 775 -+ Set perms = PosixFilePermissions.fromString("rwxrwxr-x"); -+ - if (!mLocalDataFolder.exists()) { - LOG.info("Local folder " + mLocalDataFolder + " does not exist. Creating a new one."); - mLocalDataFolder.mkdir(); - mLocalUserFolder.mkdir(); -+ Files.setPosixFilePermissions(mLocalDataFolder.toPath(), perms); -+ Files.setPosixFilePermissions(mLocalUserFolder.toPath(), perms); - return; - } - -@@ -534,6 +543,7 @@ public class WorkerStorage { - } - } - mLocalUserFolder.mkdir(); -+ Files.setPosixFilePermissions(mLocalUserFolder.toPath(), perms); - - mUnderfsOrphansFolder = mUnderfsWorkerFolder + "/orphans"; - if (!mUnderFs.exists(mUnderfsOrphansFolder)) { diff --git a/tachyon-0.5.0-gluster.patch b/tachyon-0.5.0-gluster.patch new file mode 100644 index 0000000..ac15684 --- /dev/null +++ b/tachyon-0.5.0-gluster.patch @@ -0,0 +1,2513 @@ +diff --git a/bin/tachyon b/bin/tachyon +index d062f2b..9aea6c8 100755 +--- a/bin/tachyon ++++ b/bin/tachyon +@@ -131,7 +131,7 @@ PARAMETER="" + if [ "$COMMAND" == "format" ]; then + if [ $# -eq 1 ]; then + if [ "$1" == "-s" ]; then +- if [ -e $TACHYON_UNDERFS_ADDRESS ] || [[ $TACHYON_UNDERFS_ADDRESS == hdfs://* ]] || [[ $TACHYON_UNDERFS_ADDRESS == s3://* ]]; then ++ if [ -e $TACHYON_UNDERFS_ADDRESS ] || [[ $TACHYON_UNDERFS_ADDRESS == hdfs://* ]] || [[ $TACHYON_UNDERFS_ADDRESS == s3://* ]] || [[ $TACHYON_UNDERFS_ADDRESS == glusterfs://* ]]; then + # already exists, hdfs, or s3, don't format + exit 0 + else +diff --git a/conf/tachyon-glusterfs-env.sh.template b/conf/tachyon-glusterfs-env.sh.template +new file mode 100755 +index 0000000..325246e +--- /dev/null ++++ b/conf/tachyon-glusterfs-env.sh.template +@@ -0,0 +1,57 @@ ++#!/usr/bin/env bash ++ ++# This file contains environment variables required to run Tachyon. Copy it as tachyon-env.sh and ++# edit that to configure Tachyon for your site. At a minimum, ++# the following variables should be set: ++# ++# - JAVA_HOME, to point to your JAVA installation ++# - TACHYON_MASTER_ADDRESS, to bind the master to a different IP address or hostname ++# - TACHYON_UNDERFS_ADDRESS, to set the under filesystem address. ++# - TACHYON_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker ++# - TACHYON_RAM_FOLDER, to set where worker stores in memory data ++# - TACHYON_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem, ++# org.apache.hadoop.hdfs.DistributedFileSystem) ++ ++# The following gives an example: ++ ++if [[ `uname -a` == Darwin* ]]; then ++ # Assuming Mac OS X ++ export JAVA_HOME=$(/usr/libexec/java_home) ++ export TACHYON_RAM_FOLDER=/Volumes/ramdisk ++ export TACHYON_JAVA_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc=" ++else ++ # Assuming Linux ++ if [ -z "$JAVA_HOME" ]; then ++ export JAVA_HOME=/usr/lib/jvm/java-1.6.0-sun-1.6.0.45.x86_64/ ++ fi ++ export TACHYON_RAM_FOLDER=/mnt/ramdisk ++fi ++ ++export TACHYON_MASTER_ADDRESS=localhost ++ ++export TACHYON_UNDERFS_ADDRESS=glusterfs:/// ++export TACHYON_UNDERFS_GLUSTER_VOLUMES=tachyon_vol ++export TACHYON_UNDERFS_GLUSTER_MOUNTS=/vol ++export TACHYON_UNDERFS_GLUSTER_MR_DIR=glusterfs:///mapred/system ++export TACHYON_WORKER_MEMORY_SIZE=1GB ++ ++CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ++ ++export TACHYON_JAVA_OPTS+=" ++ -Dlog4j.configuration=file:$CONF_DIR/log4j.properties ++ -Dtachyon.debug=true ++ -Dtachyon.underfs.address=$TACHYON_UNDERFS_ADDRESS ++ -Dtachyon.data.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/data ++ -Dtachyon.workers.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/workers ++ -Dtachyon.worker.memory.size=$TACHYON_WORKER_MEMORY_SIZE ++ -Dtachyon.worker.data.folder=$TACHYON_RAM_FOLDER/tachyonworker/ ++ -Dtachyon.master.worker.timeout.ms=60000 ++ -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS ++ -Dtachyon.master.journal.folder=$TACHYON_HOME/journal/ ++ -Dtachyon.master.pinlist=/pinfiles;/pindata ++ -Dtachyon.underfs.glusterfs.volumes=$TACHYON_UNDERFS_GLUSTER_VOLUMES ++ -Dtachyon.underfs.glusterfs.mounts=$TACHYON_UNDERFS_GLUSTER_MOUNTS ++ -Dtachyon.underfs.glusterfs.mapred.system.dir=$TACHYON_UNDERFS_GLUSTER_MR_DIR ++ -Dorg.apache.jasper.compiler.disablejsr199=true ++" ++export JAVA="$JAVA_HOME/bin/java $TACHYON_JAVA_OPTS" +\ No newline at end of file +diff --git a/docs/Startup-Tasks-for-New-Contributors.md b/docs/Startup-Tasks-for-New-Contributors.md +index fb72ba5..2e65c10 100644 +--- a/docs/Startup-Tasks-for-New-Contributors.md ++++ b/docs/Startup-Tasks-for-New-Contributors.md +@@ -46,13 +46,6 @@ You can generate Eclipse configure file by run: + + Then import the folder into Eclipse. + +-### Testing +- +-If you want to run unit tests, you can use command: mvn test. +- +-If you want to run a single tests, you can use command: mvn -Dtest=TestCircle#mytest test ; +-e.g. mvn -Dtest=TachyonFSTest#createFileTest test ; +- + ### Coding Style + + - Follow the style of the existing codebase. Specifically, we use +@@ -71,10 +64,8 @@ e.g. mvn -Dtest=TachyonFSTest#createFileTest test ; + - Break your work into small, single-purpose patches if possible. It’s much harder to merge in + a large change with a lot of disjoint features. + +-- Make sure that any methods you add maintain the alphabetical ordering of method names in each file. +- + - Submit the patch as a GitHub pull request. For a tutorial, see the GitHub guides on + [forking a repo](https://help.github.com/articles/fork-a-repo) and + [sending a pull request](https://help.github.com/articles/using-pull-requests). + +-- Make sure that your code passes the unit tests: mvn test. ++- Make sure that your code passes the unit tests. +diff --git a/docs/Syncing-the-Underlying-Filesystem.md b/docs/Syncing-the-Underlying-Filesystem.md +index 9fc449a..7ab9c8c 100644 +--- a/docs/Syncing-the-Underlying-Filesystem.md ++++ b/docs/Syncing-the-Underlying-Filesystem.md +@@ -3,33 +3,15 @@ layout: global + title: Syncing the Underlayer Filesystem + --- + +-Often times, there is already data in the underlying store, but when Tachyon is started, it will +-not have knowledge about the preexisting files. ++Often times, there is already data in the underlying store, but when Tachyon is started, it will not ++have knowledge about the preexisting files. + + Use the tachyon shell command loadufs to sync the filesystems. + +- $ ./bin/tachyon loadufs [TACHYON_PATH] [UNDERLYING_FILESYSTEM_PATH] [Optional EXCLUDE_PATHS] ++ $ ./bin/tachyon loadufs [TACHYON_ADDRESS] [UNDERLYING_FILESYSTEM_ADDRESS] [ROOT_DIRECTORY] [-Optional EXCLUDE_PATHS] + + For example: + +- $ ./bin/tachyon loadufs tachyon://127.0.0.1:19998 hdfs://localhost:9000 tachyon ++ $ ./bin/tachyon loadufs tachyon://127.0.0.1:19998 hdfs://localhost:9000 / /tachyon + +-Would load the meta-data for all the files in the local hdfs, except for the tachyon folder. +- +- $ ./bin/tachyon loadufs tachyon://127.0.0.1:19998/tomlogs file:///Users/tom/logs tachyon;spark +- +-Would load meta-data for all local files under the /Users/tom/logs directory (except for tachyon +-and spark) to address tachyon://127.0.0.1:19998/tomlogs. If /Users/tom/logs itself is a file, only +-that file is loaded as /tomlogs/logs in the TFS. The prefix "file://" can be safely omitted for +-a local file system. +- +-Note that the optional EXCLUDE_PATHS are prefixes relative to the given local file path. Moreover, +-only files matching the given prefixes relative to the path will be excluded. Hence, in the above +-last example, logs/tachyon and logs/spark will be excluded, but not logs/shark/tachyon nor +-logs/shark/spark. To exclude these two paths as well, the exclude list should be specified as +-"tachyon;spark;shark/tachyon;shark/spark". It is important to note that when ";" is present to +-concatenate multiple prefixes the quote marks must be used; otherwise it would be treated as +-multiple commands to be executed in tandem. +- +-In a sense, loadufs is similar to the unix mount command. It's not called mount so as not to cause +-confusions with the use of mount in the tachyon scripts. ++Would load the meta-data for all the files in the local hdfs, except for the Tachyon folder. +diff --git a/main/pom.xml b/main/pom.xml +index d5e0ea5..2db4297 100644 +--- a/main/pom.xml ++++ b/main/pom.xml +@@ -49,6 +49,13 @@ + jar + compile + ++ ++ org.apache.hadoop.fs.glusterfs ++ glusterfs-hadoop ++ 2.1.8 ++ jar ++ compile ++ + + org.eclipse.jetty + jetty-server +diff --git a/main/src/main/java/tachyon/Constants.java b/main/src/main/java/tachyon/Constants.java +index da558a8..e286d36 100644 +--- a/main/src/main/java/tachyon/Constants.java ++++ b/main/src/main/java/tachyon/Constants.java +@@ -22,7 +22,6 @@ public class Constants { + public static final int MB = KB * 1024; + public static final int GB = MB * 1024; + public static final long TB = GB * 1024L; +- public static final long PB = TB * 1024L; + + public static final String ANSI_RESET = "\u001B[0m"; + public static final String ANSI_BLACK = "\u001B[30m"; +diff --git a/main/src/main/java/tachyon/Format.java b/main/src/main/java/tachyon/Format.java +index 51ba23b..7af0819 100644 +--- a/main/src/main/java/tachyon/Format.java ++++ b/main/src/main/java/tachyon/Format.java +@@ -71,7 +71,7 @@ public class Format { + if (ufs.exists(localFolder)) { + String[] files = ufs.list(localFolder); + for (String file : files) { +- ufs.delete(CommonUtils.concat(localFolder, file), true); ++ ufs.delete(localFolder + Constants.PATH_SEPARATOR + file, true); + } + } + } else { +@@ -79,4 +79,4 @@ public class Format { + System.exit(-1); + } + } +-} ++} +\ No newline at end of file +diff --git a/main/src/main/java/tachyon/LeaderInquireClient.java b/main/src/main/java/tachyon/LeaderInquireClient.java +index defcd03..826fdc2 100644 +--- a/main/src/main/java/tachyon/LeaderInquireClient.java ++++ b/main/src/main/java/tachyon/LeaderInquireClient.java +@@ -72,7 +72,8 @@ public class LeaderInquireClient { + long maxTime = 0; + String leader = ""; + for (String master : masters) { +- Stat stat = CLIENT.checkExists().forPath(CommonUtils.concat(LEADER_PATH, master)); ++ Stat stat = ++ CLIENT.checkExists().forPath(LEADER_PATH + Constants.PATH_SEPARATOR + master); + if (stat != null && stat.getCtime() > maxTime) { + maxTime = stat.getCtime(); + leader = master; +diff --git a/main/src/main/java/tachyon/PrefixList.java b/main/src/main/java/tachyon/PrefixList.java +index 5acdd15..9528319 100644 +--- a/main/src/main/java/tachyon/PrefixList.java ++++ b/main/src/main/java/tachyon/PrefixList.java +@@ -67,18 +67,4 @@ public class PrefixList { + public boolean outList(String path) { + return !inList(path); + } +- +- /** +- * Print out all prefixes separated by ";". +- * +- * @return the string representation like "a;b/c" +- */ +- @Override +- public String toString() { +- StringBuilder s = new StringBuilder(); +- for (String prefix : LIST) { +- s.append(prefix).append(";"); +- } +- return s.toString(); +- } + } +\ No newline at end of file +diff --git a/main/src/main/java/tachyon/UnderFileSystem.java b/main/src/main/java/tachyon/UnderFileSystem.java +index 6e27464..b8e8d40 100644 +--- a/main/src/main/java/tachyon/UnderFileSystem.java ++++ b/main/src/main/java/tachyon/UnderFileSystem.java +@@ -44,49 +44,16 @@ public abstract class UnderFileSystem { + } + + public static UnderFileSystem get(String path) { +- if (path.startsWith("hdfs://") || path.startsWith("s3://") || path.startsWith("s3n://")) { ++ if (path.startsWith("hdfs://") || path.startsWith("file://") || path.startsWith("s3://") ++ || path.startsWith("s3n://") || path.startsWith("glusterfs:///")) { + return UnderFileSystemHdfs.getClient(path); +- } else if (path.startsWith(Constants.PATH_SEPARATOR) || path.startsWith("file://")) { ++ } else if (path.startsWith(Constants.PATH_SEPARATOR)) { + return UnderFileSystemSingleLocal.getClient(); + } + CommonUtils.illegalArgumentException("Unknown under file system scheme " + path); + return null; + } + +- /** +- * Transform an input string like hdfs://host:port/dir, hdfs://host:port, file:///dir, /dir +- * into a pair of address and path. The returned pairs are ("hdfs://host:port", "/dir"), +- * ("hdfs://host:port", "/"), and ("/", "/dir"), respectively. +- * +- * @param path +- * the input path string +- * @return null if s does not start with tachyon://, tachyon-ft://, hdfs://, s3://, s3n://, +- * file://, /. Or a pair of strings denoting the under FS address and the relative path +- * relative to that address. For local FS (with prefixes file:// or /), the under FS +- * address is "/" and the path starts with "/". +- */ +- public static Pair parse(String path) { +- if (path == null) { +- return null; +- } else if (path.startsWith("tachyon://") || path.startsWith("tachyon-ft://") +- || path.startsWith("hdfs://") || path.startsWith("s3://") || path.startsWith("s3n://")) { +- String prefix = path.substring(0, path.indexOf("://") + 3); +- String body = path.substring(prefix.length()); +- if (body.contains(Constants.PATH_SEPARATOR)) { +- int ind = body.indexOf(Constants.PATH_SEPARATOR); +- return new Pair(prefix + body.substring(0, ind), body.substring(ind)); +- } else { +- return new Pair(path, Constants.PATH_SEPARATOR); +- } +- } else if (path.startsWith("file://") || path.startsWith(Constants.PATH_SEPARATOR)) { +- String prefix = "file://"; +- String suffix = path.startsWith(prefix) ? path.substring(prefix.length()) : path; +- return new Pair(Constants.PATH_SEPARATOR, suffix); +- } +- +- return null; +- } +- + public abstract void close() throws IOException; + + public abstract OutputStream create(String path) throws IOException; +@@ -115,25 +82,11 @@ public abstract class UnderFileSystem { + public abstract boolean isFile(String path) throws IOException; + + /** +- * Returns an array of strings naming the files and directories in the directory denoted by this +- * abstract pathname. +- * +- *

+- * If this abstract pathname does not denote a directory, then this method returns {@code null}. +- * Otherwise an array of strings is returned, one for each file or directory in the directory. +- * Names denoting the directory itself and the directory's parent directory are not included in +- * the result. Each string is a file name rather than a complete path. +- * +- *

+- * There is no guarantee that the name strings in the resulting array will appear in any specific +- * order; they are not, in particular, guaranteed to appear in alphabetical order. ++ * List all the files in the folder. + * + * @param path + * the path to list. +- * @return An array of strings naming the files and directories in the directory denoted by this +- * abstract pathname. The array will be empty if the directory is empty. Returns +- * {@code null} if this abstract pathname does not denote a directory, or if an I/O error +- * occurs. ++ * @return all the file names under the path. + * @throws IOException + */ + public abstract String[] list(String path) throws IOException; +diff --git a/main/src/main/java/tachyon/UnderFileSystemHdfs.java b/main/src/main/java/tachyon/UnderFileSystemHdfs.java +index 7766b2e..235fa2f 100644 +--- a/main/src/main/java/tachyon/UnderFileSystemHdfs.java ++++ b/main/src/main/java/tachyon/UnderFileSystemHdfs.java +@@ -31,6 +31,8 @@ import org.apache.hadoop.fs.permission.FsPermission; + import org.apache.hadoop.hdfs.DistributedFileSystem; + import org.apache.log4j.Logger; + ++import org.apache.hadoop.fs.glusterfs.*; ++ + import tachyon.conf.CommonConf; + import tachyon.util.CommonUtils; + +@@ -56,19 +58,28 @@ public class UnderFileSystemHdfs extends UnderFileSystem { + mUfsPrefix = fsDefaultName; + Configuration tConf = new Configuration(); + tConf.set("fs.defaultFS", fsDefaultName); +- tConf.set("fs.hdfs.impl", CommonConf.get().UNDERFS_HDFS_IMPL); ++ String glusterfsPrefix = "glusterfs:///"; ++ if (fsDefaultName.startsWith(glusterfsPrefix)) { ++ tConf.set("fs.glusterfs.impl", CommonConf.get().UNDERFS_GLUSTERFS_IMPL); ++ tConf.set("mapred.system.dir", CommonConf.get().UNDERFS_GLUSTERFS_MR_DIR); ++ tConf.set("fs.glusterfs.volumes", CommonConf.get().UNDERFS_GLUSTERFS_VOLUMES); ++ tConf.set("fs.glusterfs.volume.fuse." + CommonConf.get().UNDERFS_GLUSTERFS_VOLUMES, ++ CommonConf.get().UNDERFS_GLUSTERFS_MOUNTS); ++ }else{ ++ tConf.set("fs.hdfs.impl", CommonConf.get().UNDERFS_HDFS_IMPL); + +- // To disable the instance cache for hdfs client, otherwise it causes the +- // FileSystem closed exception. Being configurable for unit/integration +- // test only, and not expose to the end-user currently. +- tConf.set("fs.hdfs.impl.disable.cache", +- System.getProperty("fs.hdfs.impl.disable.cache", "false")); ++ // To disable the instance cache for hdfs client, otherwise it causes the ++ // FileSystem closed exception. Being configurable for unit/integration ++ // test only, and not expose to the end-user currently. ++ tConf.set("fs.hdfs.impl.disable.cache", ++ System.getProperty("fs.hdfs.impl.disable.cache", "false")); ++ } + + if (System.getProperty("fs.s3n.awsAccessKeyId") != null) { +- tConf.set("fs.s3n.awsAccessKeyId", System.getProperty("fs.s3n.awsAccessKeyId")); ++ tConf.set("fs.s3n.awsAccessKeyId", System.getProperty("fs.s3n.awsAccessKeyId")); + } + if (System.getProperty("fs.s3n.awsSecretAccessKey") != null) { +- tConf.set("fs.s3n.awsSecretAccessKey", System.getProperty("fs.s3n.awsSecretAccessKey")); ++ tConf.set("fs.s3n.awsSecretAccessKey", System.getProperty("fs.s3n.awsSecretAccessKey")); + } + Path path = new Path(fsDefaultName); + mFs = path.getFileSystem(tConf); +@@ -256,8 +267,7 @@ public class UnderFileSystemHdfs extends UnderFileSystem { + String[] rtn = new String[files.length]; + int i = 0; + for (FileStatus status : files) { +- // only return the relative path, to keep consistent with java.io.File.list() +- rtn[i ++] = status.getPath().toString().substring(path.length()); // mUfsPrefix ++ rtn[i ++] = status.getPath().toString().substring(mUfsPrefix.length()); + } + return rtn; + } else { +diff --git a/main/src/main/java/tachyon/UnderFileSystemSingleLocal.java b/main/src/main/java/tachyon/UnderFileSystemSingleLocal.java +index 250a939..76e6b67 100644 +--- a/main/src/main/java/tachyon/UnderFileSystemSingleLocal.java ++++ b/main/src/main/java/tachyon/UnderFileSystemSingleLocal.java +@@ -70,7 +70,7 @@ public class UnderFileSystemSingleLocal extends UnderFileSystem { + if (recursive && file.isDirectory()) { + String[] files = file.list(); + for (String child : files) { +- success = success && delete(CommonUtils.concat(path, child), true); ++ success = success && delete(path + Constants.PATH_SEPARATOR + child, true); + } + } + +diff --git a/main/src/main/java/tachyon/Users.java b/main/src/main/java/tachyon/Users.java +index 5ab1426..0eab6b4 100644 +--- a/main/src/main/java/tachyon/Users.java ++++ b/main/src/main/java/tachyon/Users.java +@@ -77,11 +77,11 @@ public class Users { + } + + public String getUserTempFolder(long userId) { +- return CommonUtils.concat(USER_FOLDER, userId); ++ return USER_FOLDER + Constants.PATH_SEPARATOR + userId; + } + + public String getUserUnderfsTempFolder(long userId) { +- return CommonUtils.concat(USER_UNDERFS_FOLDER, userId); ++ return USER_UNDERFS_FOLDER + Constants.PATH_SEPARATOR + userId; + } + + /** +diff --git a/main/src/main/java/tachyon/client/BlockOutStream.java b/main/src/main/java/tachyon/client/BlockOutStream.java +index 355771e..c19ac2b 100644 +--- a/main/src/main/java/tachyon/client/BlockOutStream.java ++++ b/main/src/main/java/tachyon/client/BlockOutStream.java +@@ -80,7 +80,7 @@ public class BlockOutStream extends OutStream { + throw new IOException(msg); + } + +- mLocalFilePath = CommonUtils.concat(localFolder.getPath(), BLOCK_ID); ++ mLocalFilePath = localFolder.getPath() + Constants.PATH_SEPARATOR + BLOCK_ID; + mLocalFile = new RandomAccessFile(mLocalFilePath, "rw"); + mLocalFileChannel = mLocalFile.getChannel(); + // change the permission of the temporary file in order that the worker can move it. +diff --git a/main/src/main/java/tachyon/client/FileOutStream.java b/main/src/main/java/tachyon/client/FileOutStream.java +index 7395d18..5f76584 100644 +--- a/main/src/main/java/tachyon/client/FileOutStream.java ++++ b/main/src/main/java/tachyon/client/FileOutStream.java +@@ -24,7 +24,6 @@ import org.apache.log4j.Logger; + + import tachyon.Constants; + import tachyon.UnderFileSystem; +-import tachyon.util.CommonUtils; + + /** + * FileOutStream implementation of TachyonFile. It can only be gotten by +@@ -61,7 +60,7 @@ public class FileOutStream extends OutStream { + mCachedBytes = 0; + + if (WRITE_TYPE.isThrough()) { +- mUnderFsFile = CommonUtils.concat(TFS.createAndGetUserUnderfsTempFolder(), FILE.FID); ++ mUnderFsFile = TFS.createAndGetUserUnderfsTempFolder() + Constants.PATH_SEPARATOR + FILE.FID; + UnderFileSystem underfsClient = UnderFileSystem.get(mUnderFsFile); + if (BLOCK_CAPACITY > Integer.MAX_VALUE) { + throw new IOException("BLOCK_CAPCAITY (" + BLOCK_CAPACITY + ") can not bigger than " +@@ -133,10 +132,9 @@ public class FileOutStream extends OutStream { + + @Override + public void flush() throws IOException { +- // TODO We only flush the checkpoint output stream. Flush for RAMFS block streams. +- if (mCheckpointOutputStream != null) { +- mCheckpointOutputStream.flush(); +- } ++ // We only flush the checkpoint output stream. ++ // TODO flushing for RAMFS block streams. ++ mCheckpointOutputStream.flush(); + } + + private void getNextBlock() throws IOException { +diff --git a/main/src/main/java/tachyon/client/RemoteBlockInStream.java b/main/src/main/java/tachyon/client/RemoteBlockInStream.java +index 3f8a5e5..11c85f9 100644 +--- a/main/src/main/java/tachyon/client/RemoteBlockInStream.java ++++ b/main/src/main/java/tachyon/client/RemoteBlockInStream.java +@@ -30,7 +30,6 @@ import tachyon.conf.UserConf; + import tachyon.thrift.ClientBlockInfo; + import tachyon.thrift.NetAddress; + import tachyon.worker.DataServerMessage; +-import tachyon.util.CommonUtils; + + /** + * BlockInStream for remote block. +@@ -200,7 +199,8 @@ public class RemoteBlockInStream extends BlockInStream { + } + if (host.equals(InetAddress.getLocalHost().getHostName()) + || host.equals(InetAddress.getLocalHost().getHostAddress())) { +- String localFileName = CommonUtils.concat(TFS.getRootFolder(), blockInfo.blockId); ++ String localFileName = ++ TFS.getRootFolder() + Constants.PATH_SEPARATOR + blockInfo.blockId; + LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!"); + } + LOG.info(host + ":" + (port + 1) + " current host is " +diff --git a/main/src/main/java/tachyon/client/TachyonFS.java b/main/src/main/java/tachyon/client/TachyonFS.java +index bc520b9..c89ed82 100644 +--- a/main/src/main/java/tachyon/client/TachyonFS.java ++++ b/main/src/main/java/tachyon/client/TachyonFS.java +@@ -47,7 +47,6 @@ import tachyon.thrift.ClientFileInfo; + import tachyon.thrift.ClientRawTableInfo; + import tachyon.thrift.ClientWorkerInfo; + import tachyon.thrift.FileDoesNotExistException; +-import tachyon.thrift.InvalidPathException; + import tachyon.thrift.NetAddress; + import tachyon.thrift.NoWorkerException; + import tachyon.thrift.TachyonException; +@@ -131,16 +130,6 @@ public class TachyonFS { + mAvailableSpaceBytes = 0L; + } + +- /** +- * Print out the string representation of this Tachyon server address. +- * +- * @return the string representation like tachyon://host:port or tachyon-ft://host:port +- */ +- @Override +- public String toString() { +- return (mZookeeperMode ? Constants.HEADER_FT : Constants.HEADER) + mMasterAddress.toString(); +- } +- + public synchronized void accessLocalBlock(long blockId) { + connect(); + if (mWorkerClient != null && mIsWorkerLocal) { +@@ -201,21 +190,6 @@ public class TachyonFS { + } + } + +- /** +- * Cleans the given path, throwing an IOException rather than an InvalidPathException. +- * +- * @param path +- * The path to clean +- * @return the cleaned path +- */ +- private synchronized String cleanPathIOException(String path) throws IOException { +- try { +- return CommonUtils.cleanPath(path); +- } catch (InvalidPathException e) { +- throw new IOException(e.getMessage()); +- } +- } +- + public synchronized void close() throws TException { + if (mMasterClient != null) { + mMasterClient.cleanConnect(); +@@ -384,7 +358,7 @@ public class TachyonFS { + if (!mConnected) { + return -1; + } +- path = cleanPathIOException(path); ++ path = CommonUtils.cleanPath(path); + int fid = -1; + try { + fid = mMasterClient.user_createFile(path, blockSizeByte); +@@ -400,7 +374,7 @@ public class TachyonFS { + if (!mConnected) { + return -1; + } +- path = cleanPathIOException(path); ++ path = CommonUtils.cleanPath(path); + int fid = -1; + try { + fid = mMasterClient.user_createFileOnCheckpoint(path, checkpointPath); +@@ -421,7 +395,8 @@ public class TachyonFS { + if (!mConnected) { + return -1; + } +- path = cleanPathIOException(path); ++ path = CommonUtils.cleanPath(path); ++ + if (columns < 1 || columns > CommonConf.get().MAX_COLUMNS) { + throw new IOException("Column count " + columns + " is smaller than 1 or " + "bigger than " + + CommonConf.get().MAX_COLUMNS); +@@ -599,7 +574,7 @@ public class TachyonFS { + return null; + } + ClientFileInfo ret; +- path = cleanPathIOException(path); ++ path = CommonUtils.cleanPath(path); + if (useCachedMetadata && mCachedClientFileInfos.containsKey(path)) { + return mCachedClientFileInfos.get(path); + } +@@ -660,7 +635,7 @@ public class TachyonFS { + + public synchronized TachyonFile getFile(String path, boolean useCachedMetadata) + throws IOException { +- path = cleanPathIOException(path); ++ path = CommonUtils.cleanPath(path); + ClientFileInfo clientFileInfo = getClientFileInfo(path, useCachedMetadata); + if (clientFileInfo == null) { + return null; +@@ -724,7 +699,7 @@ public class TachyonFS { + return -1; + } + int fid = -1; +- path = cleanPathIOException(path); ++ path = CommonUtils.cleanPath(path); + try { + fid = mMasterClient.user_getFileId(path); + } catch (TException e) { +@@ -796,7 +771,7 @@ public class TachyonFS { + String getLocalFilename(long blockId) { + String rootFolder = getRootFolder(); + if (rootFolder != null) { +- String localFileName = CommonUtils.concat(rootFolder, blockId); ++ String localFileName = rootFolder + Constants.PATH_SEPARATOR + blockId; + File file = new File(localFileName); + if (file.exists()) { + return localFileName; +@@ -855,7 +830,7 @@ public class TachyonFS { + + public synchronized RawTable getRawTable(String path) throws IOException { + connect(); +- path = cleanPathIOException(path); ++ path = CommonUtils.cleanPath(path); + ClientRawTableInfo clientRawTableInfo; + try { + clientRawTableInfo = mMasterClient.user_getClientRawTableInfoByPath(path); +@@ -1007,7 +982,7 @@ public class TachyonFS { + if (!mConnected) { + return false; + } +- path = cleanPathIOException(path); ++ path = CommonUtils.cleanPath(path); + try { + return mMasterClient.user_mkdir(path); + } catch (TException e) { +diff --git a/main/src/main/java/tachyon/client/TachyonFile.java b/main/src/main/java/tachyon/client/TachyonFile.java +index eb09173..aab2f1f 100644 +--- a/main/src/main/java/tachyon/client/TachyonFile.java ++++ b/main/src/main/java/tachyon/client/TachyonFile.java +@@ -31,7 +31,6 @@ import tachyon.conf.UserConf; + import tachyon.thrift.ClientBlockInfo; + import tachyon.thrift.NetAddress; + import tachyon.worker.DataServerMessage; +-import tachyon.util.CommonUtils; + + /** + * Tachyon File. +@@ -108,8 +107,8 @@ public class TachyonFile implements Comparable { + * is no guarantee that the file still exists after this call returns, as Tachyon may evict blocks + * from memory at any time. + * +- * @param blockIndex +- * The index of the block in the file. ++ * @param blockId ++ * The id of the block. + * @return filename on local file system or null if file not present on local file system. + * @throws IOException + */ +@@ -121,9 +120,8 @@ public class TachyonFile implements Comparable { + + public List getLocationHosts() throws IOException { + List locations = TFS.getClientBlockInfo(FID, 0).getLocations(); +- List ret = null; ++ List ret = new ArrayList(locations.size()); + if (locations != null) { +- ret = new ArrayList(locations.size()); + for (int k = 0; k < locations.size(); k ++) { + ret.add(locations.get(k).mHost); + } +@@ -235,7 +233,7 @@ public class TachyonFile implements Comparable { + } + if (host.equals(InetAddress.getLocalHost().getHostName()) + || host.equals(InetAddress.getLocalHost().getHostAddress())) { +- String localFileName = CommonUtils.concat(TFS.getRootFolder(), FID); ++ String localFileName = TFS.getRootFolder() + Constants.PATH_SEPARATOR + FID; + LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!"); + } else { + LOG.info(host + ":" + (port + 1) + " current host is " +diff --git a/main/src/main/java/tachyon/client/table/RawColumn.java b/main/src/main/java/tachyon/client/table/RawColumn.java +index f336888..9ad11ce 100644 +--- a/main/src/main/java/tachyon/client/table/RawColumn.java ++++ b/main/src/main/java/tachyon/client/table/RawColumn.java +@@ -20,7 +20,6 @@ import tachyon.Constants; + import tachyon.client.TachyonFS; + import tachyon.client.TachyonFile; + import tachyon.master.MasterInfo; +-import tachyon.util.CommonUtils; + + /** + * The column of a RawTable. +@@ -38,8 +37,8 @@ public class RawColumn { + + // TODO creating file here should be based on id. + public boolean createPartition(int pId) throws IOException { +- return TFS.createFile(CommonUtils.concat(RAW_TABLE.getPath(), MasterInfo.COL + COLUMN_INDEX, +- pId)) > 0; ++ return TFS.createFile(RAW_TABLE.getPath() + Constants.PATH_SEPARATOR + MasterInfo.COL ++ + COLUMN_INDEX + Constants.PATH_SEPARATOR + pId) > 0; + } + + // TODO creating file here should be based on id. +@@ -49,14 +48,13 @@ public class RawColumn { + + // TODO creating file here should be based on id. + public TachyonFile getPartition(int pId, boolean cachedMetadata) throws IOException { +- return TFS.getFile( +- CommonUtils.concat(RAW_TABLE.getPath(), MasterInfo.COL + COLUMN_INDEX, pId), +- cachedMetadata); ++ return TFS.getFile(RAW_TABLE.getPath() + Constants.PATH_SEPARATOR + MasterInfo.COL ++ + COLUMN_INDEX + Constants.PATH_SEPARATOR + pId, cachedMetadata); + } + + // TODO creating file here should be based on id. + public int partitions() throws IOException { +- return TFS.getNumberOfFiles(CommonUtils.concat(RAW_TABLE.getPath(), MasterInfo.COL +- + COLUMN_INDEX)); ++ return TFS.getNumberOfFiles(RAW_TABLE.getPath() + Constants.PATH_SEPARATOR + MasterInfo.COL ++ + COLUMN_INDEX); + } +-} ++} +\ No newline at end of file +diff --git a/main/src/main/java/tachyon/conf/CommonConf.java b/main/src/main/java/tachyon/conf/CommonConf.java +index 9215d99..5e8edfc 100644 +--- a/main/src/main/java/tachyon/conf/CommonConf.java ++++ b/main/src/main/java/tachyon/conf/CommonConf.java +@@ -49,6 +49,10 @@ public class CommonConf extends Utils { + public final String UNDERFS_WORKERS_FOLDER; + + public final String UNDERFS_HDFS_IMPL; ++ public final String UNDERFS_GLUSTERFS_IMPL; ++ public final String UNDERFS_GLUSTERFS_VOLUMES; ++ public final String UNDERFS_GLUSTERFS_MOUNTS; ++ public final String UNDERFS_GLUSTERFS_MR_DIR; + public final String WEB_RESOURCES; + public final boolean USE_ZOOKEEPER; + public final String ZOOKEEPER_ADDRESS; +@@ -79,7 +83,14 @@ public class CommonConf extends Utils { + getProperty("tachyon.workers.folder", UNDERFS_ADDRESS + "/tachyon/workers"); + UNDERFS_HDFS_IMPL = + getProperty("tachyon.underfs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); +- ++ UNDERFS_GLUSTERFS_IMPL = ++ getProperty("tachyon.underfs.glusterfs.impl", "org.apache.hadoop.fs.glusterfs.GlusterFileSystem"); ++ UNDERFS_GLUSTERFS_VOLUMES = ++ getProperty("tachyon.underfs.glusterfs.volumes", null); ++ UNDERFS_GLUSTERFS_MOUNTS = ++ getProperty("tachyon.underfs.glusterfs.mounts", null); ++ UNDERFS_GLUSTERFS_MR_DIR = ++ getProperty("tachyon.underfs.glusterfs.mapred.system.dir", "glusterfs:///mapred/system"); + USE_ZOOKEEPER = getBooleanProperty("tachyon.usezookeeper", false); + if (USE_ZOOKEEPER) { + ZOOKEEPER_ADDRESS = getProperty("tachyon.zookeeper.address"); +diff --git a/main/src/main/java/tachyon/conf/MasterConf.java b/main/src/main/java/tachyon/conf/MasterConf.java +index d045199..e0dc7d9 100644 +--- a/main/src/main/java/tachyon/conf/MasterConf.java ++++ b/main/src/main/java/tachyon/conf/MasterConf.java +@@ -44,7 +44,6 @@ public class MasterConf extends Utils { + public final String FORMAT_FILE_PREFIX; + public final String HOSTNAME; + public final int PORT; +- public final String MASTER_ADDRESS; + + public final int WEB_PORT; + public final String TEMPORARY_FOLDER; +@@ -70,9 +69,6 @@ public class MasterConf extends Utils { + + HOSTNAME = getProperty("tachyon.master.hostname", "localhost"); + PORT = getIntProperty("tachyon.master.port", Constants.DEFAULT_MASTER_PORT); +- MASTER_ADDRESS = +- (CommonConf.get().USE_ZOOKEEPER ? Constants.HEADER_FT : Constants.HEADER) + HOSTNAME + ":" +- + PORT; + WEB_PORT = getIntProperty("tachyon.master.web.port", Constants.DEFAULT_MASTER_WEB_PORT); + TEMPORARY_FOLDER = getProperty("tachyon.master.temporary.folder", "/tmp"); + +diff --git a/main/src/main/java/tachyon/hadoop/TFS.java b/main/src/main/java/tachyon/hadoop/TFS.java +index 2058fb6..87661e7 100644 +--- a/main/src/main/java/tachyon/hadoop/TFS.java ++++ b/main/src/main/java/tachyon/hadoop/TFS.java +@@ -177,8 +177,7 @@ public class TFS extends FileSystem { + Path hdfsPath = Utils.getHDFSPath(path); + FileSystem fs = hdfsPath.getFileSystem(getConf()); + if (fs.exists(hdfsPath)) { +- String ufsAddrPath = CommonUtils.concat(UNDERFS_ADDRESS, path); +- UnderfsUtil.loadUnderFs(mTFS, Constants.PATH_SEPARATOR, ufsAddrPath, new PrefixList(null)); ++ UnderfsUtil.getInfo(mTFS, UNDERFS_ADDRESS, path, new PrefixList(null)); + } + } + } +diff --git a/main/src/main/java/tachyon/hadoop/Utils.java b/main/src/main/java/tachyon/hadoop/Utils.java +index 9108117..2328a51 100644 +--- a/main/src/main/java/tachyon/hadoop/Utils.java ++++ b/main/src/main/java/tachyon/hadoop/Utils.java +@@ -23,7 +23,6 @@ import org.apache.hadoop.mapred.InputSplit; + import org.apache.log4j.Logger; + + import tachyon.Constants; +-import tachyon.util.CommonUtils; + + public final class Utils { + private static final Logger LOG = Logger.getLogger(Constants.LOGGER_TYPE); +@@ -47,7 +46,7 @@ public final class Utils { + if (ret.equals("")) { + ret = path.getName(); + } else { +- ret = CommonUtils.concat(path.getName(), ret); ++ ret = path.getName() + Constants.PATH_SEPARATOR + ret; + } + path = path.getParent(); + } +diff --git a/main/src/main/java/tachyon/master/Dependency.java b/main/src/main/java/tachyon/master/Dependency.java +index 97abc8c..6a4f7e0 100644 +--- a/main/src/main/java/tachyon/master/Dependency.java ++++ b/main/src/main/java/tachyon/master/Dependency.java +@@ -13,7 +13,6 @@ import java.util.Set; + import org.apache.log4j.Logger; + + import tachyon.Constants; +-import tachyon.conf.MasterConf; + import tachyon.io.Utils; + import tachyon.thrift.ClientDependencyInfo; + import tachyon.util.CommonUtils; +@@ -125,7 +124,6 @@ public class Dependency implements ImageWriter { + // TODO We should support different types of command in the future. + // For now, assume there is only one command model. + StringBuilder sb = new StringBuilder(parseCommandPrefix()); +- sb.append(" ").append(MasterConf.get().MASTER_ADDRESS); + sb.append(" ").append(ID); + for (int k = 0; k < CHILDREN_FILES.size(); k ++) { + int id = CHILDREN_FILES.get(k); +diff --git a/main/src/main/java/tachyon/master/EditLog.java b/main/src/main/java/tachyon/master/EditLog.java +index a13a30e..102ae8d 100644 +--- a/main/src/main/java/tachyon/master/EditLog.java ++++ b/main/src/main/java/tachyon/master/EditLog.java +@@ -58,11 +58,10 @@ public class EditLog { + + public static void deleteCompletedLogs(String path, int upTo) { + UnderFileSystem ufs = UnderFileSystem.get(path); +- String folder = +- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "completed"; ++ String folder = path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; + try { + for (int i = 0; i < upTo; i ++) { +- String toDelete = CommonUtils.concat(folder, i + ".editLog"); ++ String toDelete = folder + Constants.PATH_SEPARATOR + i + ".editLog"; + LOG.info("Deleting editlog " + toDelete); + ufs.delete(toDelete, true); + } +@@ -102,18 +101,20 @@ public class EditLog { + mBackUpLogStartNum = currentLogFileNum; + int numFiles = 1; + String completedPath = +- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "completed"; ++ path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; + if (!ufs.exists(completedPath)) { + LOG.info("No completed edit logs to be parsed"); + } else { +- while (ufs.exists(CommonUtils.concat(completedPath, (completedLogs ++) + ".editLog"))) { ++ while (ufs ++ .exists(completedPath + Constants.PATH_SEPARATOR + (completedLogs ++) + ".editLog")) { + numFiles ++; + } + } + String editLogs[] = new String[numFiles]; + for (int i = 0; i < numFiles; i ++) { + if (i != numFiles - 1) { +- editLogs[i] = CommonUtils.concat(completedPath, (i + currentLogFileNum) + ".editLog"); ++ editLogs[i] = ++ completedPath + Constants.PATH_SEPARATOR + (i + currentLogFileNum) + ".editLog"; + } else { + editLogs[i] = path; + } +@@ -214,16 +215,15 @@ public class EditLog { + + public static void markUpToDate(String path) { + UnderFileSystem ufs = UnderFileSystem.get(path); +- String folder = +- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "completed"; ++ String folder = path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; + try { + // delete all loaded editlogs since mBackupLogStartNum. +- String toDelete = CommonUtils.concat(folder, mBackUpLogStartNum + ".editLog"); ++ String toDelete = folder + Constants.PATH_SEPARATOR + mBackUpLogStartNum + ".editLog"; + while (ufs.exists(toDelete)) { + LOG.info("Deleting editlog " + toDelete); + ufs.delete(toDelete, true); + mBackUpLogStartNum ++; +- toDelete = CommonUtils.concat(folder, mBackUpLogStartNum + ".editLog"); ++ toDelete = folder + Constants.PATH_SEPARATOR + mBackUpLogStartNum + ".editLog"; + } + } catch (IOException e) { + CommonUtils.runtimeException(e); +@@ -253,24 +253,24 @@ public class EditLog { + UFS = UnderFileSystem.get(path); + if (mBackUpLogStartNum != -1) { + String folder = +- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "/completed"; ++ path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; + LOG.info("Deleting completed editlogs that are part of the image."); + deleteCompletedLogs(path, mBackUpLogStartNum); + LOG.info("Backing up logs from " + mBackUpLogStartNum + " since image is not updated."); + UFS.mkdirs(folder, true); +- String toRename = CommonUtils.concat(folder, mBackUpLogStartNum + ".editLog"); ++ String toRename = folder + Constants.PATH_SEPARATOR + mBackUpLogStartNum + ".editLog"; + int currentLogFileNum = 0; + while (UFS.exists(toRename)) { +- LOG.info("Rename " + toRename + " to " +- + CommonUtils.concat(folder, currentLogFileNum + ".editLog")); ++ LOG.info("Rename " + toRename + " to " + folder + Constants.PATH_SEPARATOR ++ + currentLogFileNum + ".editLog"); + currentLogFileNum ++; + mBackUpLogStartNum ++; +- toRename = CommonUtils.concat(folder, mBackUpLogStartNum + ".editLog"); ++ toRename = folder + Constants.PATH_SEPARATOR + mBackUpLogStartNum + ".editLog"; + } + if (UFS.exists(path)) { +- UFS.rename(path, CommonUtils.concat(folder, currentLogFileNum + ".editLog")); +- LOG.info("Rename " + path + " to " +- + CommonUtils.concat(folder, currentLogFileNum + ".editLog")); ++ UFS.rename(path, folder + Constants.PATH_SEPARATOR + currentLogFileNum + ".editLog"); ++ LOG.info("Rename " + path + " to " + folder + Constants.PATH_SEPARATOR ++ + currentLogFileNum + ".editLog"); + currentLogFileNum ++; + } + mBackUpLogStartNum = -1; +@@ -500,13 +500,14 @@ public class EditLog { + _close(); + LOG.info("Edit log max size reached, rotating edit log"); + String pathPrefix = +- path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR) + 1) + "completed"; ++ path.substring(0, path.lastIndexOf(Constants.PATH_SEPARATOR)) + "/completed"; + LOG.info("path: " + path + " prefix: " + pathPrefix); + try { + if (!UFS.exists(pathPrefix)) { + UFS.mkdirs(pathPrefix, true); + } +- String newPath = CommonUtils.concat(pathPrefix, (mCurrentLogFileNum ++) + ".editLog"); ++ String newPath = ++ pathPrefix + Constants.PATH_SEPARATOR + (mCurrentLogFileNum ++) + ".editLog"; + UFS.rename(path, newPath); + LOG.info("Renamed " + path + " to " + newPath); + OS = UFS.create(path); +diff --git a/main/src/main/java/tachyon/master/Inode.java b/main/src/main/java/tachyon/master/Inode.java +index e144f33..1e04cf4 100644 +--- a/main/src/main/java/tachyon/master/Inode.java ++++ b/main/src/main/java/tachyon/master/Inode.java +@@ -108,10 +108,10 @@ public abstract class Inode implements Comparable, ImageWriter { + + @Override + public synchronized String toString() { +- StringBuilder sb = new StringBuilder("Inode("); ++ StringBuilder sb = new StringBuilder("INode("); + sb.append("ID:").append(mId).append(", NAME:").append(mName); + sb.append(", PARENT_ID:").append(mParentId); + sb.append(", CREATION_TIME_MS:").append(CREATION_TIME_MS).append(")"); + return sb.toString(); + } +-} ++} +\ No newline at end of file +diff --git a/main/src/main/java/tachyon/master/InodeFolder.java b/main/src/main/java/tachyon/master/InodeFolder.java +index 16fc58f..db3d853 100644 +--- a/main/src/main/java/tachyon/master/InodeFolder.java ++++ b/main/src/main/java/tachyon/master/InodeFolder.java +@@ -20,8 +20,8 @@ import java.io.IOException; + import java.util.ArrayList; + import java.util.HashSet; + import java.util.List; ++import java.util.Map; + import java.util.Set; +-import java.util.Collections; + + import tachyon.io.Utils; + import tachyon.thrift.ClientFileInfo; +@@ -45,14 +45,9 @@ public class InodeFolder extends Inode { + int parentId = is.readInt(); + + int numberOfChildren = is.readInt(); +- Inode[] children = new Inode[numberOfChildren]; ++ int[] children = new int[numberOfChildren]; + for (int k = 0; k < numberOfChildren; k ++) { +- byte type = is.readByte(); +- if (type == Image.T_INODE_FILE) { +- children[k] = InodeFile.loadImage(is); +- } else { +- children[k] = InodeFolder.loadImage(is); +- } ++ children[k] = is.readInt(); + } + + InodeFolder folder = new InodeFolder(fileName, fileId, parentId, creationTimeMs); +@@ -60,41 +55,22 @@ public class InodeFolder extends Inode { + return folder; + } + +- private Set mChildren = new HashSet(); ++ private Set mChildren = new HashSet(); + + public InodeFolder(String name, int id, int parentId, long creationTimeMs) { + super(name, id, parentId, true, creationTimeMs); + } + +- /** +- * Adds the given inode to the set of children. +- * +- * @param child +- * The inode to add +- */ +- public synchronized void addChild(Inode child) { +- mChildren.add(child); ++ public synchronized void addChild(int childId) { ++ mChildren.add(childId); + } + +- /** +- * Adds the given inodes to the set of children. +- * +- * @param children +- * The inodes to add +- */ +- public synchronized void addChildren(Inode[] children) { +- for (Inode child : children) { +- addChild(child); ++ public synchronized void addChildren(int[] childrenIds) { ++ for (int k = 0; k < childrenIds.length; k ++) { ++ addChild(childrenIds[k]); + } + } + +- /** +- * Generates client file info for the folder. +- * +- * @param path +- * The path of the folder in the filesystem +- * @return the generated ClientFileInfo +- */ + @Override + public ClientFileInfo generateClientFileInfo(String path) { + ClientFileInfo ret = new ClientFileInfo(); +@@ -117,91 +93,37 @@ public class InodeFolder extends Inode { + return ret; + } + +- /** +- * Returns the child with the given id. +- * +- * @param fid +- * The id of the child +- * @return the inode with the given id, or null if there is no child with that id +- */ +- public synchronized Inode getChild(int fid) { +- for (Inode child : mChildren) { +- if (child.getId() == fid) { +- return child; +- } +- } +- return null; +- } +- +- /** +- * Returns the child with the given name. +- * +- * @param name +- * The name of the child +- * @return the inode with the given name, or null if there is no child with that name +- */ +- public synchronized Inode getChild(String name) { +- for (Inode child : mChildren) { +- if (child.getName().equals(name)) { +- return child; ++ public synchronized Inode getChild(String name, Map allInodes) { ++ Inode tInode = null; ++ for (int childId : mChildren) { ++ tInode = allInodes.get(childId); ++ if (tInode != null && tInode.getName().equals(name)) { ++ return tInode; + } + } + return null; + } + +- /** +- * Returns the folder's children. +- * +- * @return an unmodifiable set of the children inodes. +- */ +- public synchronized Set getChildren() { +- return Collections.unmodifiableSet(mChildren); +- } +- +- /** +- * Returns the ids of the children. +- * +- * @return the ids of the children +- */ + public synchronized List getChildrenIds() { + List ret = new ArrayList(mChildren.size()); +- for (Inode child : mChildren) { +- ret.add(child.getId()); +- } ++ ret.addAll(mChildren); + return ret; + } + +- /** +- * Returns the number of children the folder has. +- * +- * @return the number of children in the folder. +- */ + public synchronized int getNumberOfChildren() { + return mChildren.size(); + } + +- /** +- * Removes the given inode from the folder. +- * +- * @param child +- * The Inode to remove +- * @return true if the inode was removed, false otherwise. +- */ +- public synchronized boolean removeChild(Inode child) { +- return mChildren.remove(child); ++ public synchronized void removeChild(int id) { ++ mChildren.remove(id); + } + +- /** +- * Removes the given child from the folder. +- * +- * @param name +- * The name of the Inode to remove. +- * @return true if the inode was removed, false otherwise. +- */ +- public synchronized boolean removeChild(String name) { +- for (Inode child : mChildren) { +- if (child.getName().equals(name)) { +- mChildren.remove(child); ++ public synchronized boolean removeChild(String name, Map allInodes) { ++ Inode tInode = null; ++ for (int childId : mChildren) { ++ tInode = allInodes.get(childId); ++ if (tInode != null && tInode.getName().equals(name)) { ++ mChildren.remove(childId); + return true; + } + } +@@ -215,12 +137,6 @@ public class InodeFolder extends Inode { + return sb.toString(); + } + +- /** +- * Write an image of the folder. +- * +- * @param os +- * The output stream to write the folder to +- */ + @Override + public void writeImage(DataOutputStream os) throws IOException { + os.writeByte(Image.T_INODE_FOLDER); +@@ -232,8 +148,8 @@ public class InodeFolder extends Inode { + + List children = getChildrenIds(); + os.writeInt(children.size()); +- for (Inode inode : getChildren()) { +- inode.writeImage(os); ++ for (int k = 0; k < children.size(); k ++) { ++ os.writeInt(children.get(k)); + } + } +-} ++} +\ No newline at end of file +diff --git a/main/src/main/java/tachyon/master/MasterInfo.java b/main/src/main/java/tachyon/master/MasterInfo.java +index af885e6..44675ad 100644 +--- a/main/src/main/java/tachyon/master/MasterInfo.java ++++ b/main/src/main/java/tachyon/master/MasterInfo.java +@@ -122,8 +122,7 @@ public class MasterInfo implements ImageWriter { + dep.addLostFile(tFile.getId()); + LOG.info("File " + tFile.getId() + " got lost from worker " + worker.getId() + + " . Trying to recompute it using dependency " + dep.ID); +- String path = getPath(tFile); +- if (path != null && !path.startsWith(MASTER_CONF.TEMPORARY_FOLDER)) { ++ if (!getPath(tFile).startsWith(MASTER_CONF.TEMPORARY_FOLDER)) { + mMustRecomputeDependencies.add(depId); + } + } +@@ -239,8 +238,8 @@ public class MasterInfo implements ImageWriter { + private Map mDependencies = new HashMap(); + private RawTables mRawTables = new RawTables(); + +- // TODO add initialization part for master failover or restart. All operations on these members +- // are synchronized on mDependencies. ++ // TODO add initialization part for master failover or restart. ++ // All operations on these members are synchronized on mDependencies. + private Set mUncheckpointedDependencies = new HashSet(); + private Set mPriorityDependencies = new HashSet(); + private Set mLostFiles = new HashSet(); +@@ -344,21 +343,11 @@ public class MasterInfo implements ImageWriter { + * Internal API. + * + * @param recursive +- * If recursive is true and the filesystem tree is not filled in all the way to path yet, +- * it fills in the missing components. + * @param path +- * The path to create + * @param directory +- * If true, creates an InodeFolder instead of an Inode + * @param blockSizeByte +- * If it's a file, the block size for the Inode + * @param creationTimeMs +- * The time the file was created +- * @param id +- * If not -1, use this id as the id to the inode we create at the given path. +- * Any intermediate directories created will use mInodeCounter, so using this parameter +- * assumes no intermediate directories will be created. +- * @return the id of the inode created at the given path ++ * @return + * @throws FileAlreadyExistException + * @throws InvalidPathException + * @throws BlockInfoException +@@ -378,9 +367,6 @@ public class MasterInfo implements ImageWriter { + synchronized (mRoot) { + Inode inode = getInode(pathNames); + if (inode != null) { +- if (inode.isDirectory() && directory) { +- return inode.getId(); +- } + LOG.info("FileAlreadyExistException: File " + path + " already exist."); + throw new FileAlreadyExistException("Path " + path + " already exist."); + } +@@ -435,7 +421,7 @@ public class MasterInfo implements ImageWriter { + } + + mInodes.put(ret.getId(), ret); +- ((InodeFolder) inode).addChild(ret); ++ ((InodeFolder) inode).addChild(ret.getId()); + + LOG.debug("createFile: File Created: " + ret + " parent: " + inode); + return ret.getId(); +@@ -473,7 +459,7 @@ public class MasterInfo implements ImageWriter { + } + + InodeFolder parent = (InodeFolder) mInodes.get(inode.getParentId()); +- parent.removeChild(inode); ++ parent.removeChild(inode.getId()); + mInodes.remove(inode.getId()); + if (inode.isFile()) { + String checkpointPath = ((InodeFile) inode).getCheckpointPath(); +@@ -614,43 +600,6 @@ public class MasterInfo implements ImageWriter { + } + + /** +- * Adds the given inode to mFileIdPinList, traversing folders recursively. Used while loading an +- * image. +- * +- * @param inode +- * The inode to add +- */ +- private void addToFileIdPinList(Inode inode) throws IOException { +- if (inode.isFile() && ((InodeFile) inode).isPin()) { +- synchronized (mFileIdPinList) { +- mFileIdPinList.add(inode.getId()); +- } +- } else if (inode.isDirectory()) { +- for (Inode child : ((InodeFolder) inode).getChildren()) { +- addToFileIdPinList(child); +- } +- } +- } +- +- /** +- * While loading an image, addToInodeMap will map the various ids to their inodes. +- * +- * @param inode +- * The inode to add +- * @param map +- * The map to add the inodes to +- */ +- private void addToInodeMap(Inode inode, Map map) { +- map.put(inode.getId(), inode); +- if (inode.isDirectory()) { +- InodeFolder inodeFolder = (InodeFolder) inode; +- for (Inode child : inodeFolder.getChildren()) { +- addToInodeMap(child, map); +- } +- } +- } +- +- /** + * A worker cache a block in its memory. + * + * @param workerId +@@ -827,7 +776,7 @@ public class MasterInfo implements ImageWriter { + } + + for (int k = 0; k < columns; k ++) { +- mkdir(CommonUtils.concat(path, COL + k)); ++ mkdir(path + Constants.PATH_SEPARATOR + COL + k); + } + + return id; +@@ -1116,6 +1065,23 @@ public class MasterInfo implements ImageWriter { + } + + /** ++ * Get the path of a file with the given id ++ * ++ * @param fileId ++ * The id of the file to look up ++ * @return the path of the file ++ */ ++ public String getFileNameById(int fileId) throws FileDoesNotExistException { ++ synchronized (mRoot) { ++ Inode inode = mInodes.get(fileId); ++ if (inode == null) { ++ throw new FileDoesNotExistException("FileId " + fileId + " does not exist"); ++ } ++ return getPath(inode); ++ } ++ } ++ ++ /** + * Get the file id's of the given paths. It recursively scans directories for the file id's inside + * of them. + * +@@ -1153,11 +1119,14 @@ public class MasterInfo implements ImageWriter { + } + + if (inode.isDirectory()) { +- Set children = ((InodeFolder) inode).getChildren(); ++ List childernIds = ((InodeFolder) inode).getChildrenIds(); + ++ if (!path.endsWith(Constants.PATH_SEPARATOR)) { ++ path += Constants.PATH_SEPARATOR; ++ } + synchronized (mRoot) { +- for (Inode child : children) { +- ret.add(getClientFileInfo(child.getId())); ++ for (int k : childernIds) { ++ ret.add(getClientFileInfo(k)); + } + } + } else { +@@ -1183,9 +1152,10 @@ public class MasterInfo implements ImageWriter { + InodeFolder tFolder = tPair.getFirst(); + String curPath = tPair.getSecond(); + +- Set children = tFolder.getChildren(); +- for (Inode tInode : children) { +- String newPath = CommonUtils.concat(curPath, tInode.getName()); ++ List childrenIds = tFolder.getChildrenIds(); ++ for (int id : childrenIds) { ++ Inode tInode = mInodes.get(id); ++ String newPath = curPath + Constants.PATH_SEPARATOR + tInode.getName(); + if (tInode.isDirectory()) { + nodesQueue.add(new Pair((InodeFolder) tInode, newPath)); + } else if (((InodeFile) tInode).isFullyInMemory()) { +@@ -1236,7 +1206,7 @@ public class MasterInfo implements ImageWriter { + if (cur.isFile()) { + return null; + } +- cur = ((InodeFolder) cur).getChild(name); ++ cur = ((InodeFolder) cur).getChild(name, mInodes); + } + + return cur; +@@ -1304,24 +1274,8 @@ public class MasterInfo implements ImageWriter { + if (inode.getParentId() == 1) { + return Constants.PATH_SEPARATOR + inode.getName(); + } +- return CommonUtils.concat(getPath(mInodes.get(inode.getParentId())), inode.getName()); +- } +- } +- +- /** +- * Get the path of a file with the given id +- * +- * @param fileId +- * The id of the file to look up +- * @return the path of the file +- */ +- public String getPath(int fileId) throws FileDoesNotExistException { +- synchronized (mRoot) { +- Inode inode = mInodes.get(fileId); +- if (inode == null) { +- throw new FileDoesNotExistException("FileId " + fileId + " does not exist"); +- } +- return getPath(inode); ++ return getPath(mInodes.get(inode.getParentId())) + Constants.PATH_SEPARATOR ++ + inode.getName(); + } + } + +@@ -1475,7 +1429,7 @@ public class MasterInfo implements ImageWriter { + * If true, select a random worker + * @param host + * If random is false, select a worker on this host +- * @return the address of the selected worker, or null if no address could be found ++ * @return the address of the selected worker + */ + public NetAddress getWorker(boolean random, String host) { + synchronized (mWorkers) { +@@ -1593,15 +1547,17 @@ public class MasterInfo implements ImageWriter { + if (inode.isFile()) { + ret.add(inode.getId()); + } else if (recursive) { +- Queue queue = new LinkedList(); +- queue.addAll(((InodeFolder) inode).getChildren()); ++ Queue queue = new LinkedList(); ++ queue.addAll(((InodeFolder) inode).getChildrenIds()); + + while (!queue.isEmpty()) { +- Inode qinode = queue.poll(); +- if (qinode.isDirectory()) { +- queue.addAll(((InodeFolder) inode).getChildren()); ++ int id = queue.poll(); ++ inode = mInodes.get(id); ++ ++ if (inode.isDirectory()) { ++ queue.addAll(((InodeFolder) inode).getChildrenIds()); + } else { +- ret.add(qinode.getId()); ++ ret.add(id); + } + } + } +@@ -1649,16 +1605,15 @@ public class MasterInfo implements ImageWriter { + inode = InodeFolder.loadImage(is); + } + ++ LOG.info("Putting " + inode); + if (inode.getId() > mInodeCounter.get()) { + mInodeCounter.set(inode.getId()); + } + +- addToInodeMap(inode, mInodes); +- addToFileIdPinList(inode); +- + if (inode.getId() == 1) { + mRoot = (InodeFolder) inode; + } ++ mInodes.put(inode.getId(), inode); + } else if (Image.T_RAW_TABLE == type) { + mRawTables.loadImage(is); + } else { +@@ -1689,15 +1644,22 @@ public class MasterInfo implements ImageWriter { + if (inode.isFile()) { + ret.add(path); + } else { +- Set children = ((InodeFolder) inode).getChildren(); ++ List childrenIds = ((InodeFolder) inode).getChildrenIds(); ++ ++ if (!path.endsWith(Constants.PATH_SEPARATOR)) { ++ path += Constants.PATH_SEPARATOR; ++ } + ret.add(path); + + synchronized (mRoot) { +- for (Inode child : children) { +- if (recursive) { +- ret.addAll(ls(CommonUtils.concat(path, child.getName()), true)); +- } else { +- ret.add(CommonUtils.concat(path, child.getName())); ++ for (int k : childrenIds) { ++ inode = mInodes.get(k); ++ if (inode != null) { ++ if (recursive) { ++ ret.addAll(ls(path + inode.getName(), true)); ++ } else { ++ ret.add(path + inode.getName()); ++ } + } + } + } +@@ -1711,7 +1673,7 @@ public class MasterInfo implements ImageWriter { + * + * @param path + * The path to create a directory at +- * @return true if and only if the directory was created; false otherwise ++ * @return true if the creation was successful and false if it wasn't + */ + public boolean mkdir(String path) throws FileAlreadyExistException, InvalidPathException, + TachyonException { +@@ -1836,9 +1798,9 @@ public class MasterInfo implements ImageWriter { + + srcInode.setName(dstName); + InodeFolder parent = (InodeFolder) mInodes.get(srcInode.getParentId()); +- parent.removeChild(srcInode); ++ parent.removeChild(srcInode.getId()); + srcInode.setParentId(dstFolderInode.getId()); +- ((InodeFolder) dstFolderInode).addChild(srcInode); ++ ((InodeFolder) dstFolderInode).addChild(srcInode.getId()); + + mJournal.getEditLog().rename(srcInode.getId(), dstPath); + mJournal.getEditLog().flush(); +@@ -2055,13 +2017,30 @@ public class MasterInfo implements ImageWriter { + * The output stream to write the image to + */ + public void writeImage(DataOutputStream os) throws IOException { ++ Queue folderQueue = new LinkedList(); ++ + synchronized (mRoot) { +- synchronized (mDependencies) { +- for (Dependency dep : mDependencies.values()) { +- dep.writeImage(os); +- } ++ for (Dependency dep : mDependencies.values()) { ++ dep.writeImage(os); + } ++ + mRoot.writeImage(os); ++ folderQueue.add(mRoot); ++ while (!folderQueue.isEmpty()) { ++ List childrenIds = folderQueue.poll().getChildrenIds(); ++ for (int id : childrenIds) { ++ Inode tInode = mInodes.get(id); ++ tInode.writeImage(os); ++ if (tInode.isDirectory()) { ++ folderQueue.add((InodeFolder) tInode); ++ } else if (((InodeFile) tInode).isPin()) { ++ synchronized (mFileIdPinList) { ++ mFileIdPinList.add(tInode.getId()); ++ } ++ } ++ } ++ } ++ + mRawTables.writeImage(os); + + os.writeByte(Image.T_CHECKPOINT); +diff --git a/main/src/main/java/tachyon/util/CommonUtils.java b/main/src/main/java/tachyon/util/CommonUtils.java +index cd43ccf..4b326c6 100644 +--- a/main/src/main/java/tachyon/util/CommonUtils.java ++++ b/main/src/main/java/tachyon/util/CommonUtils.java +@@ -19,7 +19,6 @@ import java.io.IOException; + import java.io.InputStream; + import java.io.OutputStream; + import java.io.PrintStream; +-import java.math.BigDecimal; + import java.net.InetSocketAddress; + import java.nio.ByteBuffer; + import java.text.DateFormat; +@@ -29,7 +28,6 @@ import java.util.Date; + import java.util.List; + import java.util.Scanner; + +-import org.apache.commons.io.FilenameUtils; + import org.apache.log4j.Logger; + + import tachyon.Constants; +@@ -81,17 +79,14 @@ public final class CommonUtils { + changeLocalFilePermission(filePath, "777"); + } + +- /** +- * Checks and normalizes the given path +- * +- * @param path +- * The path to clean up +- * @return a normalized version of the path, with single separators between path components and +- * dot components resolved +- */ +- public static String cleanPath(String path) throws InvalidPathException { +- validatePath(path); +- return FilenameUtils.separatorsToUnix(FilenameUtils.normalizeNoEndSeparator(path)); ++ public static String cleanPath(String path) throws IOException { ++ if (path == null || path.isEmpty()) { ++ throw new IOException("Path (" + path + ") is invalid."); ++ } ++ while (path.endsWith(Constants.PATH_SEPARATOR) && path.length() > 1) { ++ path = path.substring(0, path.length() - 1); ++ } ++ return path; + } + + public static ByteBuffer cloneByteBuffer(ByteBuffer buf) { +@@ -109,31 +104,6 @@ public final class CommonUtils { + return ret; + } + +- /** +- * Add the path component to the base path +- * +- * @param args +- * The components to concatenate +- * @return the concatenated path +- */ +- public static String concat(Object... args) { +- if (args.length == 0) { +- return ""; +- } +- String retPath = args[0].toString(); +- for (int k = 1; k < args.length; k ++) { +- while (retPath.endsWith(Constants.PATH_SEPARATOR)) { +- retPath = retPath.substring(0, retPath.length() - 1); +- } +- if (args[k].toString().startsWith(Constants.PATH_SEPARATOR)) { +- retPath += args[k].toString(); +- } else { +- retPath += Constants.PATH_SEPARATOR + args[k].toString(); +- } +- } +- return retPath; +- } +- + public static String convertByteArrayToStringWithoutEscape(byte[] data) { + StringBuilder sb = new StringBuilder(data.length); + for (int i = 0; i < data.length; i ++) { +@@ -214,7 +184,8 @@ public final class CommonUtils { + * @return the name of the file + */ + public static String getName(String path) throws InvalidPathException { +- return FilenameUtils.getName(cleanPath(path)); ++ String[] pathNames = getPathComponents(path); ++ return pathNames[pathNames.length - 1]; + } + + /** +@@ -225,8 +196,8 @@ public final class CommonUtils { + * @return the path split into components + */ + public static String[] getPathComponents(String path) throws InvalidPathException { +- path = cleanPath(path); +- if (isRoot(path)) { ++ validatePath(path); ++ if (path.length() == 1 && path.equals(Constants.PATH_SEPARATOR)) { + String[] ret = new String[1]; + ret[0] = ""; + return ret; +@@ -260,17 +231,6 @@ public final class CommonUtils { + throw new IllegalArgumentException(msg); + } + +- /** +- * Check if the given path is the root. +- * +- * @param path +- * The path to check +- * @return true if the path is the root +- */ +- public static boolean isRoot(String path) throws InvalidPathException { +- return Constants.PATH_SEPARATOR.equals(cleanPath(path)); +- } +- + public static String listToString(List list) { + StringBuilder sb = new StringBuilder(); + for (int k = 0; k < list.size(); k ++) { +@@ -340,11 +300,6 @@ public final class CommonUtils { + return (long) (ret * Constants.GB + alpha); + } else if (end.equals("tb")) { + return (long) (ret * Constants.TB + alpha); +- } else if (end.equals("pb")) { +- // When parsing petabyte values, we can't multiply with doubles and longs, since that will +- // lose presicion with such high numbers. Therefore we use a BigDecimal. +- BigDecimal PBDecimal = new BigDecimal(Constants.PB); +- return PBDecimal.multiply(BigDecimal.valueOf(ret)).longValue(); + } else { + runtimeException("Fail to parse " + ori + " as memory size"); + return -1; +@@ -437,8 +392,8 @@ public final class CommonUtils { + } + + public static void validatePath(String path) throws InvalidPathException { +- if (path == null || path.isEmpty() || !path.startsWith(Constants.PATH_SEPARATOR) +- || path.contains(" ")) { ++ if (path == null || !path.startsWith(Constants.PATH_SEPARATOR) ++ || (path.length() > 1 && path.endsWith(Constants.PATH_SEPARATOR)) || path.contains(" ")) { + throw new InvalidPathException("Path " + path + " is invalid."); + } + } +diff --git a/main/src/main/java/tachyon/util/UnderfsUtil.java b/main/src/main/java/tachyon/util/UnderfsUtil.java +index bd8bfdd..75b25fc 100644 +--- a/main/src/main/java/tachyon/util/UnderfsUtil.java ++++ b/main/src/main/java/tachyon/util/UnderfsUtil.java +@@ -18,14 +18,19 @@ import java.io.IOException; + import java.util.LinkedList; + import java.util.Queue; + ++import org.apache.hadoop.conf.Configuration; + import org.apache.log4j.Logger; ++import org.apache.thrift.TException; + + import tachyon.Constants; +-import tachyon.Pair; + import tachyon.PrefixList; + import tachyon.UnderFileSystem; + import tachyon.Version; + import tachyon.client.TachyonFS; ++import tachyon.thrift.FileAlreadyExistException; ++import tachyon.thrift.FileDoesNotExistException; ++import tachyon.thrift.InvalidPathException; ++import tachyon.thrift.SuspectedFileSizeException; + + /** + * Utilities related to under filesystem +@@ -33,167 +38,73 @@ import tachyon.client.TachyonFS; + public class UnderfsUtil { + private static Logger LOG = Logger.getLogger(Constants.LOGGER_TYPE); + +- /** +- * Create a new path relative to a given TFS root. +- * +- * @param tfsRootPath +- * the destination point in TFS to load the under FS path onto +- * @param ufsRootPath +- * the source path in the under FS to be loaded +- * @param path +- * the path relative to ufsRootPath of a file to be loaded +- * @return the new path relative to tfsRootPath. +- */ +- private static String createTFSPath(String tfsRootPath, String ufsRootPath, String path) { +- String filePath = path.substring(ufsRootPath.length()); +- if (filePath.isEmpty()) { +- // retrieve the basename in ufsRootPath +- filePath = path.substring(ufsRootPath.lastIndexOf(Constants.PATH_SEPARATOR) + 1); +- } +- return CommonUtils.concat(tfsRootPath, filePath); +- } +- +- /** +- * Load files under path "ufsAddrRootPath" (excluding excludePathPrefix relative to the path) +- * to the given tfs under a given destination path. +- * +- * @param tfsAddrRootPath +- * the TFS address and path to load the src files, like "tachyon://host:port/dest". +- * @param ufsAddrRootPath +- * the address and root path of the under FS, like "hdfs://host:port/src". +- * @param excludePaths +- * paths to exclude from ufsRootPath, which will not be loaded in TFS. +- * @throws IOException +- */ +- public static void loadUnderFs(String tfsAddrRootPath, String ufsAddrRootPath, +- String excludePaths) throws IOException { +- Pair tfsPair = UnderFileSystem.parse(tfsAddrRootPath); +- String tfsAddress = tfsPair.getFirst(); +- String tfsRootPath = tfsPair.getSecond(); +- TachyonFS tfs = TachyonFS.get(tfsAddress); +- +- PrefixList excludePathPrefix = new PrefixList(excludePaths, ";"); +- +- loadUnderFs(tfs, tfsRootPath, ufsAddrRootPath, excludePathPrefix); +- } +- +- /** +- * Load files under path "ufsAddress/ufsRootPath" (excluding excludePathPrefix) +- * to the given tfs under the given tfsRootPath directory. +- * +- * @param tfs +- * the TFS handler created out of address like "tachyon://host:port" +- * @param tfsRootPath +- * the destination point in TFS to load the under FS path onto +- * @param ufsAddrRootPath +- * the address and root path of the under FS, like "hdfs://host:port/dir". +- * @param excludePathPrefix +- * paths to exclude from ufsRootPath, which will not be registered in TFS. +- * @throws IOException +- */ +- public static void loadUnderFs(TachyonFS tfs, String tfsRootPath, String ufsAddrRootPath, ++ public static void getInfo(TachyonFS tfs, String underfsAddress, String rootPath, + PrefixList excludePathPrefix) throws IOException { +- LOG.info(tfs + tfsRootPath + " " + ufsAddrRootPath + " " + excludePathPrefix); +- +- Pair tfsPair = UnderFileSystem.parse(ufsAddrRootPath); +- String ufsAddress = tfsPair.getFirst(); +- String ufsRootPath = tfsPair.getSecond(); ++ LOG.info(tfs + " " + underfsAddress + " " + rootPath + " " + excludePathPrefix); + +- if (!tfs.exist(tfsRootPath)) { +- tfs.mkdir(tfsRootPath); +- // TODO Add the following. +- // if (tfs.mkdir(tfsRootPath)) { +- // LOG.info("directory " + tfsRootPath + " does not exist in Tachyon: created"); +- // } else { +- // throw new IOException("Failed to create folder in Tachyon: " + tfsRootPath); +- // } +- } +- +- // create the under FS handler (e.g. hdfs, local FS, s3 etc.) +- UnderFileSystem ufs = UnderFileSystem.get(ufsAddress); ++ Configuration tConf = new Configuration(); ++ tConf.set("fs.default.name", underfsAddress + rootPath); ++ // TODO Use underfs to make this generic. ++ UnderFileSystem fs = UnderFileSystem.get(underfsAddress); + +- Queue ufsPathQueue = new LinkedList(); +- if (excludePathPrefix.outList(ufsRootPath)) { +- ufsPathQueue.add(ufsAddrRootPath); ++ Queue pathQueue = new LinkedList(); ++ if (excludePathPrefix.outList(rootPath)) { ++ pathQueue.add(underfsAddress + rootPath); + } +- +- while (!ufsPathQueue.isEmpty()) { +- String ufsPath = ufsPathQueue.poll(); // the absolute path +- if (ufs.isFile(ufsPath)) { +- String tfsPath = createTFSPath(tfsRootPath, ufsAddrRootPath, ufsPath); +- if (tfs.exist(tfsPath)) { +- LOG.info("File " + tfsPath + " already exists in Tachyon."); ++ while (!pathQueue.isEmpty()) { ++ String path = pathQueue.poll(); ++ if (fs.isFile(path)) { ++ String filePath = path.substring(underfsAddress.length()); ++ if (tfs.exist(filePath)) { ++ LOG.info("File " + filePath + " already exists in Tachyon."); + continue; + } +- int fileId = tfs.createFile(tfsPath, ufsPath); ++ int fileId = tfs.createFile(filePath, path); + if (fileId == -1) { +- LOG.info("Failed to create tachyon file: " + tfsPath); ++ LOG.info("Failed to create tachyon file: " + filePath); + } else { +- LOG.info("Create tachyon file " + tfsPath + " with file id " + fileId + " and " +- + "checkpoint location " + ufsPath); ++ LOG.info("Create tachyon file " + filePath + " with file id " + fileId + " and " ++ + "checkpoint location " + path); + } +- } else { // ufsPath is a directory +- String[] files = ufs.list(ufsPath); // ufs.list() returns relative path ++ } else { ++ String[] files = fs.list(path); + if (files != null) { + for (String filePath : files) { + LOG.info("Get: " + filePath); +- String aPath = CommonUtils.concat(ufsPath, filePath); +- String checkPath = aPath.substring(ufsAddrRootPath.length()); +- if (checkPath.startsWith(Constants.PATH_SEPARATOR)) { +- checkPath = checkPath.substring(Constants.PATH_SEPARATOR.length()); +- } +- if (excludePathPrefix.inList(checkPath)) { +- LOG.info("excluded: " + checkPath); +- } else { +- ufsPathQueue.add(aPath); ++ if (excludePathPrefix.outList(filePath)) { ++ pathQueue.add(underfsAddress + filePath); + } + } + } +- String tfsPath = createTFSPath(tfsRootPath, ufsAddrRootPath, ufsPath); +- if (!tfs.exist(tfsPath)) { +- tfs.mkdir(tfsPath); +- // TODO Add the following. +- // if (tfs.mkdir(tfsPath)) { +- // LOG.info("Created tachyon folder " + tfsPath + " with checkpoint location " + ufsPath); +- // } else { +- // LOG.info("Failed to create tachyon folder: " + tfsPath); +- // } ++ String filePath = path.substring(underfsAddress.length()); ++ if (!tfs.exist(filePath)) { ++ tfs.mkdir(filePath); + } + } + } + } + +- public static void main(String[] args) { +- if (!(args.length == 2 || args.length == 3)) { +- printUsage(); ++ public static void main(String[] args) throws SuspectedFileSizeException, InvalidPathException, ++ IOException, FileDoesNotExistException, FileAlreadyExistException, TException { ++ if (!(args.length == 3 || args.length == 4)) { ++ String prefix = ++ "java -cp target/tachyon-" + Version.VERSION + "-jar-with-dependencies.jar " ++ + "tachyon.util.UnderfsUtil "; ++ System.out.println("Usage: " + prefix + " " ++ + "[]"); ++ System.out.println("Example: " + prefix ++ + "tachyon://127.0.0.1:19998 hdfs://localhost:9000 / /tachyon"); + System.exit(-1); + } + +- String exList = (args.length == 3) ? args[2] : ""; +- +- try { +- loadUnderFs(args[0], args[1], exList); +- } catch (Exception e) { +- e.printStackTrace(); +- printUsage(); +- System.exit(-2); ++ PrefixList tExcludePathPrefix = null; ++ if (args.length == 4) { ++ tExcludePathPrefix = new PrefixList(args[3], ";"); ++ } else { ++ tExcludePathPrefix = new PrefixList(null); + } + ++ getInfo(TachyonFS.get(args[0]), args[1], args[2], tExcludePathPrefix); + System.exit(0); + } +- +- public static void printUsage() { +- String cmd = +- "java -cp target/tachyon-" + Version.VERSION + "-jar-with-dependencies.jar " +- + "tachyon.util.UnderfsUtil "; +- +- System.out.println("Usage: " + cmd + " " +- + "[]"); +- System.out +- .println("Example: " + cmd + "tachyon://127.0.0.1:19998/a hdfs://localhost:9000/b c"); +- System.out.println("Example: " + cmd + "tachyon://127.0.0.1:19998/a file:///b c"); +- System.out.println("Example: " + cmd + "tachyon://127.0.0.1:19998/a /b c"); +- System.out.print("In the TFS, all files under local FS /b will be registered under /a, "); +- System.out.println("except for those with prefix c"); +- } + } +diff --git a/main/src/main/java/tachyon/web/WebInterfaceBrowseServlet.java b/main/src/main/java/tachyon/web/WebInterfaceBrowseServlet.java +index 6b1917a..c0d0ebf 100644 +--- a/main/src/main/java/tachyon/web/WebInterfaceBrowseServlet.java ++++ b/main/src/main/java/tachyon/web/WebInterfaceBrowseServlet.java +@@ -363,8 +363,9 @@ public class WebInterfaceBrowseServlet extends HttpServlet { + String currentPath = Constants.PATH_SEPARATOR; + pathInfos[0] = new UiFileInfo(mMasterInfo.getClientFileInfo(currentPath)); + for (int i = 1; i < splitPath.length - 1; i ++) { +- currentPath = CommonUtils.concat(currentPath, splitPath[i]); ++ currentPath = currentPath + splitPath[i]; + pathInfos[i] = new UiFileInfo(mMasterInfo.getClientFileInfo(currentPath)); ++ currentPath = currentPath + Constants.PATH_SEPARATOR; + } + request.setAttribute("pathInfos", pathInfos); + return; +diff --git a/main/src/main/java/tachyon/web/WebInterfaceDependencyServlet.java b/main/src/main/java/tachyon/web/WebInterfaceDependencyServlet.java +index d7474bb..787e981 100644 +--- a/main/src/main/java/tachyon/web/WebInterfaceDependencyServlet.java ++++ b/main/src/main/java/tachyon/web/WebInterfaceDependencyServlet.java +@@ -35,10 +35,10 @@ public class WebInterfaceDependencyServlet extends HttpServlet { + try { + ClientDependencyInfo dependencyInfo = mMasterInfo.getClientDependencyInfo(dependencyId); + for (int pId : dependencyInfo.parents) { +- parentFileNames.add(mMasterInfo.getPath(pId)); ++ parentFileNames.add(mMasterInfo.getFileNameById(pId)); + } + for (int cId : dependencyInfo.children) { +- childrenFileNames.add(mMasterInfo.getPath(cId)); ++ childrenFileNames.add(mMasterInfo.getFileNameById(cId)); + } + } catch (DependencyDoesNotExistException ddnee) { + request.setAttribute("error", ddnee.getMessage()); +diff --git a/main/src/main/java/tachyon/worker/DataServerMessage.java b/main/src/main/java/tachyon/worker/DataServerMessage.java +index de01bb3..04b6a06 100644 +--- a/main/src/main/java/tachyon/worker/DataServerMessage.java ++++ b/main/src/main/java/tachyon/worker/DataServerMessage.java +@@ -77,7 +77,7 @@ public class DataServerMessage { + throw new IOException("Length can not be negative except -1: " + len); + } + +- String filePath = CommonUtils.concat(WorkerConf.get().DATA_FOLDER, blockId); ++ String filePath = WorkerConf.get().DATA_FOLDER + Constants.PATH_SEPARATOR + blockId; + ret.LOG.info("Try to response remote requst by reading from " + filePath); + RandomAccessFile file = new RandomAccessFile(filePath, "r"); + +diff --git a/main/src/main/java/tachyon/worker/WorkerStorage.java b/main/src/main/java/tachyon/worker/WorkerStorage.java +index d0e089f..a02fdf3 100644 +--- a/main/src/main/java/tachyon/worker/WorkerStorage.java ++++ b/main/src/main/java/tachyon/worker/WorkerStorage.java +@@ -174,8 +174,9 @@ public class WorkerStorage { + + // TODO checkpoint process. In future, move from midPath to dstPath should be done by + // master +- String midPath = CommonUtils.concat(mUnderfsWorkerDataFolder, fileId); +- String dstPath = CommonUtils.concat(CommonConf.get().UNDERFS_DATA_FOLDER, fileId); ++ String midPath = mUnderfsWorkerDataFolder + Constants.PATH_SEPARATOR + fileId; ++ String dstPath = ++ CommonConf.get().UNDERFS_DATA_FOLDER + Constants.PATH_SEPARATOR + fileId; + LOG.info("Thread " + ID + " is checkpointing file " + fileId + " from " + + mLocalDataFolder.toString() + " to " + midPath + " to " + dstPath); + +@@ -196,7 +197,8 @@ public class WorkerStorage { + long fileSizeByte = 0; + for (int k = 0; k < fileInfo.blockIds.size(); k ++) { + File tempFile = +- new File(CommonUtils.concat(mLocalDataFolder.toString(), fileInfo.blockIds.get(k))); ++ new File(mLocalDataFolder.toString() + Constants.PATH_SEPARATOR ++ + fileInfo.blockIds.get(k)); + fileSizeByte += tempFile.length(); + InputStream is = new FileInputStream(tempFile); + byte[] buf = new byte[16 * Constants.KB]; +@@ -313,7 +315,8 @@ public class WorkerStorage { + mLocalDataFolder = new File(dataFolder); + mLocalUserFolder = + new File(mLocalDataFolder.toString(), WorkerConf.get().USER_TEMP_RELATIVE_FOLDER); +- mUnderfsWorkerFolder = CommonUtils.concat(COMMON_CONF.UNDERFS_WORKERS_FOLDER, mWorkerId); ++ mUnderfsWorkerFolder = ++ COMMON_CONF.UNDERFS_WORKERS_FOLDER + Constants.PATH_SEPARATOR + mWorkerId; + mUnderfsWorkerDataFolder = mUnderfsWorkerFolder + "/data"; + mUnderFs = UnderFileSystem.get(COMMON_CONF.UNDERFS_ADDRESS); + mUsers = new Users(mLocalUserFolder.toString(), mUnderfsWorkerFolder); +@@ -359,8 +362,8 @@ public class WorkerStorage { + public void addCheckpoint(long userId, int fileId) throws FileDoesNotExistException, + SuspectedFileSizeException, FailedToCheckpointException, BlockInfoException, TException { + // TODO This part need to be changed. +- String srcPath = CommonUtils.concat(getUserUnderfsTempFolder(userId), fileId); +- String dstPath = CommonUtils.concat(COMMON_CONF.UNDERFS_DATA_FOLDER, fileId); ++ String srcPath = getUserUnderfsTempFolder(userId) + Constants.PATH_SEPARATOR + fileId; ++ String dstPath = COMMON_CONF.UNDERFS_DATA_FOLDER + Constants.PATH_SEPARATOR + fileId; + try { + if (!mUnderFs.rename(srcPath, dstPath)) { + throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath); +@@ -403,8 +406,8 @@ public class WorkerStorage { + + public void cacheBlock(long userId, long blockId) throws FileDoesNotExistException, + SuspectedFileSizeException, BlockInfoException, TException { +- File srcFile = new File(CommonUtils.concat(getUserTempFolder(userId), blockId)); +- File dstFile = new File(CommonUtils.concat(mLocalDataFolder, blockId)); ++ File srcFile = new File(getUserTempFolder(userId) + Constants.PATH_SEPARATOR + blockId); ++ File dstFile = new File(mLocalDataFolder + Constants.PATH_SEPARATOR + blockId); + long fileSizeBytes = srcFile.length(); + if (!srcFile.exists()) { + throw new FileDoesNotExistException("File " + srcFile + " does not exist."); +@@ -453,21 +456,21 @@ public class WorkerStorage { + * The block to be removed. + * @return Removed file size in bytes. + */ +- private long freeBlock(long blockId) { ++ private synchronized long freeBlock(long blockId) { + Long freedFileBytes = null; +- synchronized (mLatestBlockAccessTimeMs) { +- if (mBlockSizes.containsKey(blockId)) { +- mWorkerSpaceCounter.returnUsedBytes(mBlockSizes.get(blockId)); +- File srcFile = new File(CommonUtils.concat(mLocalDataFolder, blockId)); +- srcFile.delete(); ++ if (mBlockSizes.containsKey(blockId)) { ++ mWorkerSpaceCounter.returnUsedBytes(mBlockSizes.get(blockId)); ++ File srcFile = new File(mLocalDataFolder + Constants.PATH_SEPARATOR + blockId); ++ srcFile.delete(); ++ synchronized (mLatestBlockAccessTimeMs) { + mLatestBlockAccessTimeMs.remove(blockId); + freedFileBytes = mBlockSizes.remove(blockId); + mRemovedBlockList.add(blockId); + mMemoryData.remove(blockId); +- LOG.info("Removed Data " + blockId); +- } else { +- LOG.warn("File " + blockId + " does not exist in memory."); + } ++ LOG.info("Removed Data " + blockId); ++ } else { ++ LOG.warn("File " + blockId + " does not exist in memory."); + } + + return freedFileBytes == null ? 0 : freedFileBytes; +@@ -705,7 +708,7 @@ public class WorkerStorage { + RandomAccessFile localFile = new RandomAccessFile(file, "r"); + ByteBuffer buf = localFile.getChannel().map(MapMode.READ_ONLY, 0, file.length()); + +- String ufsOrphanBlock = CommonUtils.concat(mUnderfsOrphansFolder, blockId); ++ String ufsOrphanBlock = mUnderfsOrphansFolder + Constants.PATH_SEPARATOR + blockId; + OutputStream os = mUnderFs.create(ufsOrphanBlock); + int BULKSIZE = 1024 * 64; + byte[] bulk = new byte[BULKSIZE]; +diff --git a/main/src/main/webapp/browse.jsp b/main/src/main/webapp/browse.jsp +index d73a10e..5281511 100644 +--- a/main/src/main/webapp/browse.jsp ++++ b/main/src/main/webapp/browse.jsp +@@ -1,7 +1,5 @@ + <%@ page import="java.util.*" %> + <%@ page import="tachyon.web.*" %> +-<%@ page import="static org.apache.commons.lang.StringEscapeUtils.escapeHtml" %> +-<%@ page import="static java.net.URLEncoder.encode" %> + + + +@@ -35,11 +33,11 @@ +

+ +@@ -115,7 +113,7 @@ + <% if (!fileInfo.getIsDirectory()) { %> + + <% } %> +- "><%= escapeHtml(fileInfo.getName()) %> ++ <%= fileInfo.getName() %> + + <%= fileInfo.getSize() %> + <%= fileInfo.getBlockSizeBytes() %> +diff --git a/main/src/main/webapp/viewFile.jsp b/main/src/main/webapp/viewFile.jsp +index 2eb564e..31ce58b 100644 +--- a/main/src/main/webapp/viewFile.jsp ++++ b/main/src/main/webapp/viewFile.jsp +@@ -1,7 +1,5 @@ + <%@ page import="java.util.*" %> + <%@ page import="tachyon.web.*" %> +-<%@ page import="static org.apache.commons.lang.StringEscapeUtils.escapeHtml" %> +-<%@ page import="static java.net.URLEncoder.encode" %> + + + +@@ -17,7 +15,7 @@ + function displayContent() + { + var tmp = document.getElementById("offset").value; +- window.location.href = "./browse?path=<%= encode(request.getAttribute("currentPath").toString(), "UTF-8") %>&offset=" + tmp; ++ window.location.href = "./browse?path=<%= request.getAttribute("currentPath") %>&offset=" + tmp; + } + +
+@@ -38,7 +36,7 @@ +

+ <%= request.getAttribute("invalidPathError") %> +

+-

<%= escapeHtml(request.getAttribute("currentPath").toString()) %>: First 5KB from <%= request.getAttribute("viewingOffset") %> in ASCII

++

<%= request.getAttribute("currentPath") %>: First 5KB from <%= request.getAttribute("viewingOffset") %> in ASCII

+ +
+ +diff --git a/main/src/test/java/tachyon/PrefixListTest.java b/main/src/test/java/tachyon/PrefixListTest.java +index 3b31644..8d06b0d 100644 +--- a/main/src/test/java/tachyon/PrefixListTest.java ++++ b/main/src/test/java/tachyon/PrefixListTest.java +@@ -83,19 +83,4 @@ public final class PrefixListTest { + Assert.assertTrue(prefixList.outList("")); + Assert.assertTrue(prefixList.outList(null)); + } +- +- @Test +- public void toStringTest() { +- PrefixList prefixList = new PrefixList("", ";"); +- Assert.assertEquals(prefixList.toString(), ";"); +- +- prefixList = new PrefixList(";", ";"); +- Assert.assertEquals(prefixList.toString(), ""); +- +- prefixList = new PrefixList("a/b;c", ";"); +- Assert.assertEquals(prefixList.toString(), "a/b;c;"); +- +- prefixList = new PrefixList("a/b;c;", ";"); +- Assert.assertEquals(prefixList.toString(), "a/b;c;"); +- } + } +diff --git a/main/src/test/java/tachyon/UnderFileSystemCluster.java b/main/src/test/java/tachyon/UnderFileSystemCluster.java +index 2ca049e..387283e 100644 +--- a/main/src/test/java/tachyon/UnderFileSystemCluster.java ++++ b/main/src/test/java/tachyon/UnderFileSystemCluster.java +@@ -109,7 +109,7 @@ public abstract class UnderFileSystemCluster { + String path = getUnderFilesystemAddress() + Constants.PATH_SEPARATOR; + UnderFileSystem ufs = UnderFileSystem.get(path); + for (String p : ufs.list(path)) { +- ufs.delete(CommonUtils.concat(path, p), true); ++ ufs.delete(path + p, true); + } + } + } +diff --git a/main/src/test/java/tachyon/UnderFileSystemTest.java b/main/src/test/java/tachyon/UnderFileSystemTest.java +deleted file mode 100644 +index 087694e..0000000 +--- a/main/src/test/java/tachyon/UnderFileSystemTest.java ++++ /dev/null +@@ -1,67 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one or more +- * contributor license agreements. See the NOTICE file distributed with +- * this work for additional information regarding copyright ownership. +- * The ASF licenses this file to You under the Apache License, Version 2.0 +- * (the "License"); you may not use this file except in compliance with +- * the License. You may obtain a copy of the License at +- * http://www.apache.org/licenses/LICENSE-2.0 +- * Unless required by applicable law or agreed to in writing, software +- * distributed under the License is distributed on an "AS IS" BASIS, +- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- * See the License for the specific language governing permissions and +- * limitations under the License. +- */ +-package tachyon; +- +-import org.junit.Assert; +-import org.junit.Test; +- +-/** +- * Unit tests for {@link UnderFileSystem} +- */ +-public final class UnderFileSystemTest { +- +- @Test +- public void parseTest() { +- Pair result = UnderFileSystem.parse("/path"); +- Assert.assertEquals(result.getFirst(), "/"); +- Assert.assertEquals(result.getSecond(), "/path"); +- +- result = UnderFileSystem.parse("file:///path"); +- Assert.assertEquals(result.getFirst(), "/"); +- Assert.assertEquals(result.getSecond(), "/path"); +- +- result = UnderFileSystem.parse("tachyon://localhost:19998"); +- Assert.assertEquals(result.getFirst(), "tachyon://localhost:19998"); +- Assert.assertEquals(result.getSecond(), "/"); +- +- result = UnderFileSystem.parse("tachyon://localhost:19998/"); +- Assert.assertEquals(result.getFirst(), "tachyon://localhost:19998"); +- Assert.assertEquals(result.getSecond(), "/"); +- +- result = UnderFileSystem.parse("tachyon://localhost:19998/path"); +- Assert.assertEquals(result.getFirst(), "tachyon://localhost:19998"); +- Assert.assertEquals(result.getSecond(), "/path"); +- +- result = UnderFileSystem.parse("tachyon-ft://localhost:19998/path"); +- Assert.assertEquals(result.getFirst(), "tachyon-ft://localhost:19998"); +- Assert.assertEquals(result.getSecond(), "/path"); +- +- result = UnderFileSystem.parse("hdfs://localhost:19998/path"); +- Assert.assertEquals(result.getFirst(), "hdfs://localhost:19998"); +- Assert.assertEquals(result.getSecond(), "/path"); +- +- result = UnderFileSystem.parse("s3://localhost:19998/path"); +- Assert.assertEquals(result.getFirst(), "s3://localhost:19998"); +- Assert.assertEquals(result.getSecond(), "/path"); +- +- result = UnderFileSystem.parse("s3n://localhost:19998/path"); +- Assert.assertEquals(result.getFirst(), "s3n://localhost:19998"); +- Assert.assertEquals(result.getSecond(), "/path"); +- +- Assert.assertEquals(UnderFileSystem.parse(null), null); +- Assert.assertEquals(UnderFileSystem.parse(""), null); +- Assert.assertEquals(UnderFileSystem.parse("anythingElse"), null); +- } +-} +diff --git a/main/src/test/java/tachyon/client/TachyonFSTest.java b/main/src/test/java/tachyon/client/TachyonFSTest.java +index ab54ff8..f99afb5 100644 +--- a/main/src/test/java/tachyon/client/TachyonFSTest.java ++++ b/main/src/test/java/tachyon/client/TachyonFSTest.java +@@ -338,13 +338,6 @@ public class TachyonFSTest { + } + + @Test +- public void toStringTest() throws Exception { +- String tfsAddress = "tachyon://localhost:19998"; +- TachyonFS tfs = TachyonFS.get(tfsAddress); +- Assert.assertEquals(tfs.toString(), "tachyon://localhost/127.0.0.1:19998"); +- } +- +- @Test + public void unlockBlockTest1() throws IOException { + TachyonFile tFile = null; + int numOfFiles = 5; +diff --git a/main/src/test/java/tachyon/command/TFsShellTest.java b/main/src/test/java/tachyon/command/TFsShellTest.java +index cb3a05a..a3b887e 100644 +--- a/main/src/test/java/tachyon/command/TFsShellTest.java ++++ b/main/src/test/java/tachyon/command/TFsShellTest.java +@@ -363,10 +363,10 @@ public class TFsShellTest { + Assert.assertTrue(tFile.isDirectory()); + } + +- @Test ++ @Test(expected = IOException.class) + public void mkdirExistingTest() throws IOException { +- Assert.assertEquals(0, mFsShell.mkdir(new String[] { "mkdir", "/testFile1" })); +- Assert.assertEquals(0, mFsShell.mkdir(new String[] { "mkdir", "/testFile1" })); ++ mFsShell.mkdir(new String[] { "mkdir", "/testFile1" }); ++ mFsShell.mkdir(new String[] { "mkdir", "/testFile1" }); + } + + @Test(expected = IOException.class) +diff --git a/main/src/test/java/tachyon/hadoop/GlusterFSTest.java b/main/src/test/java/tachyon/hadoop/GlusterFSTest.java +new file mode 100644 +index 0000000..2ef6a74 +--- /dev/null ++++ b/main/src/test/java/tachyon/hadoop/GlusterFSTest.java +@@ -0,0 +1,55 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package tachyon.hadoop; ++ ++import java.io.IOException; ++ ++import org.apache.hadoop.fs.glusterfs.*; ++import org.junit.After; ++import org.junit.Before; ++import org.junit.Assert; ++import org.junit.Test; ++ ++import tachyon.UnderFileSystem; ++ ++/** ++ * Unit tests for tachyon.hadoop.GlusterFS. ++ */ ++public class GlusterFSTest { ++ private UnderFileSystem mHcfs = null; ++ ++ @After ++ public final void after() throws Exception { ++ System.clearProperty("fs.default.name"); ++ System.clearProperty("tachyon.underfs.glusterfs.mapred.system.dir"); ++ System.clearProperty("tachyon.underfs.glusterfs.mounts"); ++ System.clearProperty("tachyon.underfs.glusterfs.volumes"); ++ } ++ ++ @Before ++ public final void before() throws IOException { ++ System.setProperty("fs.default.name", "glusterfs:///"); ++ System.setProperty("tachyon.underfs.glusterfs.mapred.system.dir", "glusterfs:///mapred/system"); ++ System.setProperty("tachyon.underfs.glusterfs.mounts", "/vol"); ++ System.setProperty("tachyon.underfs.glusterfs.volumes", "tachyon_vol"); ++ } ++ @Test ++ public void createGlusterFS() throws Exception { ++ mHcfs = UnderFileSystem.get("glusterfs:///"); ++ ++ Assert.assertTrue(mHcfs.create("tachyon_test") != null); ++ Assert.assertTrue(mHcfs.delete("tachyon_test", false)); ++ } ++} +diff --git a/main/src/test/java/tachyon/hadoop/HadoopCompatibleFSTest.java b/main/src/test/java/tachyon/hadoop/HadoopCompatibleFSTest.java +new file mode 100644 +index 0000000..359ee97 +--- /dev/null ++++ b/main/src/test/java/tachyon/hadoop/HadoopCompatibleFSTest.java +@@ -0,0 +1,59 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package tachyon.hadoop; ++ ++import java.io.IOException; ++ ++// import org.apache.hadoop.conf.Configuration; ++// import org.apache.hadoop.fs.FileSystem; ++// import org.apache.hadoop.hdfs.DistributedFileSystem; ++// import org.apache.hadoop.hdfs.MiniDFSCluster; ++// import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; ++import org.junit.After; ++import org.junit.Before; ++import tachyon.client.TachyonFS; ++import tachyon.master.LocalTachyonCluster; ++ ++/** ++ * Unit tests for tachyon.hadoop.HadoopCompatibleFS. ++ */ ++public class HadoopCompatibleFSTest { ++ private LocalTachyonCluster mLocalTachyonCluster = null; ++ private TachyonFS mTfs = null; ++ ++ // private MiniDFSCluster mDfsCluster = null; ++ // private DistributedFileSystem mDfs = null; ++ ++ @After ++ public final void after() throws Exception { ++ mLocalTachyonCluster.stop(); ++ System.clearProperty("tachyon.user.quota.unit.bytes"); ++ } ++ ++ @Before ++ public final void before() throws IOException { ++ System.setProperty("tachyon.user.quota.unit.bytes", "1000"); ++ mLocalTachyonCluster = new LocalTachyonCluster(10000); ++ mLocalTachyonCluster.start(); ++ mTfs = mLocalTachyonCluster.getClient(); ++ ++ // Configuration conf = new Configuration(); ++ // System.setProperty("fs.default.name", "hdfs://localhost:54310"); ++ // mDfsCluster = new MiniDFSCluster(); ++ // FileSystem fs = mDfsCluster.getFileSystem(); ++ // Assert.assertTrue("Not a HDFS: "+fs.getUri(), fs instanceof DistributedFileSystem); ++ // mDfs = (DistributedFileSystem) fs; ++ } ++} +diff --git a/main/src/test/java/tachyon/master/InodeFolderTest.java b/main/src/test/java/tachyon/master/InodeFolderTest.java +index 3009032..4b5ea94 100644 +--- a/main/src/test/java/tachyon/master/InodeFolderTest.java ++++ b/main/src/test/java/tachyon/master/InodeFolderTest.java +@@ -31,10 +31,8 @@ public class InodeFolderTest { + @Test + public void addChildrenTest() { + InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); +- InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); +- InodeFile inodeFile2 = new InodeFile("testFile2", 3, 1, 1000, System.currentTimeMillis()); +- inodeFolder.addChild(inodeFile1); +- inodeFolder.addChild(inodeFile2); ++ inodeFolder.addChild(2); ++ inodeFolder.addChild(3); + Assert.assertEquals(2, (int) inodeFolder.getChildrenIds().get(0)); + Assert.assertEquals(3, (int) inodeFolder.getChildrenIds().get(1)); + } +@@ -44,12 +42,14 @@ public class InodeFolderTest { + InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); + InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); + InodeFile inodeFile2 = new InodeFile("testFile2", 3, 1, 1000, System.currentTimeMillis()); +- InodeFile inodeFile3 = new InodeFile("testFile3", 4, 1, 1000, System.currentTimeMillis()); +- inodeFolder.addChild(inodeFile1); +- inodeFolder.addChild(inodeFile2); +- inodeFolder.addChild(inodeFile3); ++ inodeFolder.addChild(2); ++ inodeFolder.addChild(3); ++ inodeFolder.addChild(4); ++ Map testMap = new HashMap(2); ++ testMap.put(2, inodeFile1); ++ testMap.put(3, inodeFile2); + Assert.assertEquals(3, inodeFolder.getNumberOfChildren()); +- inodeFolder.removeChild("testFile1"); ++ inodeFolder.removeChild("testFile1", testMap); + Assert.assertEquals(2, inodeFolder.getNumberOfChildren()); + Assert.assertFalse(inodeFolder.getChildrenIds().contains(2)); + } +@@ -90,21 +90,18 @@ public class InodeFolderTest { + @Test + public void removeChildTest() { + InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); +- InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); +- inodeFolder.addChild(inodeFile1); ++ inodeFolder.addChild(2); + Assert.assertEquals(1, inodeFolder.getNumberOfChildren()); +- inodeFolder.removeChild(inodeFile1); ++ inodeFolder.removeChild(2); + Assert.assertEquals(0, inodeFolder.getNumberOfChildren()); + } + + @Test + public void removeNonExistentChildTest() { + InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); +- InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); +- InodeFile inodeFile2 = new InodeFile("testFile2", 3, 1, 1000, System.currentTimeMillis()); +- inodeFolder.addChild(inodeFile1); ++ inodeFolder.addChild(2); + Assert.assertEquals(1, inodeFolder.getNumberOfChildren()); +- inodeFolder.removeChild(inodeFile2); ++ inodeFolder.removeChild(3); + Assert.assertEquals(1, inodeFolder.getNumberOfChildren()); + } + +@@ -118,9 +115,8 @@ public class InodeFolderTest { + @Test + public void sameIdChildrenTest() { + InodeFolder inodeFolder = new InodeFolder("testFolder1", 1, 0, System.currentTimeMillis()); +- InodeFile inodeFile1 = new InodeFile("testFile1", 2, 1, 1000, System.currentTimeMillis()); +- inodeFolder.addChild(inodeFile1); +- inodeFolder.addChild(inodeFile1); ++ inodeFolder.addChild(2); ++ inodeFolder.addChild(2); + Assert.assertTrue(inodeFolder.getChildrenIds().get(0) == 2); + Assert.assertEquals(1, inodeFolder.getNumberOfChildren()); + } +diff --git a/main/src/test/java/tachyon/util/CommonUtilsTest.java b/main/src/test/java/tachyon/util/CommonUtilsTest.java +index b8bb69b..b4154b2 100644 +--- a/main/src/test/java/tachyon/util/CommonUtilsTest.java ++++ b/main/src/test/java/tachyon/util/CommonUtilsTest.java +@@ -52,12 +52,5 @@ public class CommonUtilsTest { + Assert.assertEquals(k * Constants.TB / 10, CommonUtils.parseMemorySize(k / 10.0 + "TB")); + Assert.assertEquals(k * Constants.TB / 10, CommonUtils.parseMemorySize(k / 10.0 + "tB")); + } +- // We stop the pb test before 8192, since 8192 petabytes is beyond the scope of a java long. +- for (long k = 0; k < 8192; k ++) { +- Assert.assertEquals(k * Constants.PB / 10, CommonUtils.parseMemorySize(k / 10.0 + "pb")); +- Assert.assertEquals(k * Constants.PB / 10, CommonUtils.parseMemorySize(k / 10.0 + "Pb")); +- Assert.assertEquals(k * Constants.PB / 10, CommonUtils.parseMemorySize(k / 10.0 + "PB")); +- Assert.assertEquals(k * Constants.PB / 10, CommonUtils.parseMemorySize(k / 10.0 + "pB")); +- } + } + } +diff --git a/main/src/test/java/tachyon/util/UnderfsUtilTest.java b/main/src/test/java/tachyon/util/UnderfsUtilTest.java +index 7a71f3a..860a598 100644 +--- a/main/src/test/java/tachyon/util/UnderfsUtilTest.java ++++ b/main/src/test/java/tachyon/util/UnderfsUtilTest.java +@@ -31,6 +31,8 @@ import tachyon.UnderFileSystem; + import tachyon.UnderFileSystemCluster; + import tachyon.client.TachyonFS; + import tachyon.master.LocalTachyonCluster; ++import tachyon.util.UnderfsUtil; ++import tachyon.util.CommonUtils; + + /** + * To test the utilities related to under filesystem, including loadufs and etc. +@@ -62,7 +64,7 @@ public class UnderfsUtilTest { + } + + @Test +- public void loadUnderFsTest() throws IOException { ++ public void getInfoTest() throws IOException { + if (!UnderFileSystemCluster.isUFSHDFS()) { + return; + } +@@ -82,8 +84,8 @@ public class UnderfsUtilTest { + CommonUtils.touch(mUnderfsAddress + inclusion + "/1"); + } + +- UnderfsUtil.loadUnderFs(mTfs, Constants.PATH_SEPARATOR, mUnderfsAddress +- + Constants.PATH_SEPARATOR, new PrefixList(Arrays.asList(exclusions))); ++ UnderfsUtil.getInfo(mTfs, mUnderfsAddress, Constants.PATH_SEPARATOR, ++ new PrefixList(Arrays.asList(exclusions))); + + List paths = null; + for (String exclusion : exclusions) { +diff --git a/pom.xml b/pom.xml +index c74d6bb..767fbaf 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -70,6 +70,16 @@ + false + + ++ ++ glusterfs-hadoop-repo ++ http://rhbd.s3.amazonaws.com/maven/repositories/internal/ ++ ++ true ++ ++ ++ false ++ ++ + + +