From 2f748dbf4cf46943b55efe227d610762f81feae0 Mon Sep 17 00:00:00 2001 From: Timothy St. Clair Date: Feb 17 2014 22:14:37 +0000 Subject: Update to latest sources with fixes --- diff --git a/.gitignore b/.gitignore index 790d665..7448851 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /tachyon-0.4.0-515c2c8.tar.gz /tachyon-0.4.0-e5cba5b.tar.gz +/tachyon-0.4.1-4b9c806.tar.gz diff --git a/amplab-tachyon.spec b/amplab-tachyon.spec index 974ad08..4ce4cb6 100644 --- a/amplab-tachyon.spec +++ b/amplab-tachyon.spec @@ -1,12 +1,12 @@ -%global commit 520c15109e9651c3cd95a563941ad821c8fb5503 +%global commit 4b9c806bbce73db39e010a8bafd031c9b9749882 %global shortcommit %(c=%{commit}; echo ${c:0:7}) %global shortname tachyon Name: amplab-%{shortname} # Given the naming conflicts with other packages, and eventually this will -# switch to apache-tachyon should -Version: 0.4.0 -Release: 7.SNAPSHOT.%{shortcommit}%{?dist} +# switch to apache-tachyon should +Version: 0.4.1 +Release: 1.SNAPSHOT.%{shortcommit}%{?dist} Summary: Reliable file sharing at memory speed across cluster frameworks License: ASL 2.0 URL: http://tachyon-project.org/ @@ -20,6 +20,9 @@ Source5: %{shortname}-env.sh Patch0: tachyon-0.4.0-SNAPSHOT-log4props.patch Patch1: tachyon-0.4.0-defaults.patch +# Will likely go away https://github.com/amplab/tachyon/pull/128 +Patch2: tachyon-0.4.1-permissions.patch + BuildRequires: java-devel BuildRequires: mvn(commons-io:commons-io) BuildRequires: mvn(log4j:log4j) @@ -80,6 +83,7 @@ find -name '*.jar' -print -delete %patch0 -p1 %patch1 -p1 +%patch2 -p1 sed -i "s|hadoop-client|hadoop-mapreduce-client-core|" pom.xml @@ -96,9 +100,6 @@ sed -i "s|hadoop-client|hadoop-mapreduce-client-core|" p %pom_add_dep org.apache.hadoop:hadoop-common %pom_add_dep org.apache.hadoop:hadoop-hdfs -# Disabled b/c upstream has updated to 0.9.0 -# ./bin/tachyon thriftGen - %build %mvn_file org.tachyonproject:%{shortname} %{shortname} @@ -168,6 +169,7 @@ mkdir -p -m0755 %{buildroot}/%{_datadir}/hadoop/common/lib %doc LICENSE ############################################ +# NOTE: uses hdfs user vs. creating own hdfs required #%%pre #getent group tachyon >/dev/null || groupadd -f -r tachyon #if ! getent passwd tachyon >/dev/null ; then @@ -186,8 +188,8 @@ mkdir -p -m0755 %{buildroot}/%{_datadir}/hadoop/common/lib %systemd_postun_with_restart %{shortname}-slave.service %{shortname}-master.service %changelog -* Tue Jan 28 2014 Timothy St. Clair 0.4.0-7.SNAPSHOT.520c151 -- Update to latest +* Mon Feb 17 2014 Timothy St. Clair 0.4.1-1.SNAPSHOT.4b9c806 +- Update to latest, with updated fixes for rawhide * Fri Dec 6 2013 Timothy St. Clair 0.4.0-6.SNAPSHOT.8e4e9a4 - Update to latest modifications diff --git a/sources b/sources index cc5539b..0914e16 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -02bb0a1ea465b20e6b04cfd8843a681e tachyon-0.4.0-520c151.tar.gz +0e1c55d5a5bd5a768bcb1066e5f74120 tachyon-0.4.1-4b9c806.tar.gz diff --git a/tachyon-0.4.0-defaults.patch b/tachyon-0.4.0-defaults.patch index 45ebb37..1ed43d6 100644 --- a/tachyon-0.4.0-defaults.patch +++ b/tachyon-0.4.0-defaults.patch @@ -1,5 +1,5 @@ diff --git a/bin/tachyon b/bin/tachyon -index d795ed0..7d1ce67 100755 +index 9122bcd..66cb2a2 100755 --- a/bin/tachyon +++ b/bin/tachyon @@ -1,7 +1,7 @@ @@ -24,22 +24,22 @@ index d795ed0..7d1ce67 100755 fi if [[ "$1" == "Basic" ]]; then -- $bin/tachyon tfs rm /Basic_File_$2 -+ $bin/tachyon.sh tfs rm /Basic_File_$2 - $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicOperations $MASTER_ADDRESS:19998 /BasicFile_$2 $2 +- $bin/tachyon tfs rm /BasicFile_$2 ++ $bin/tachyon.sh tfs rm /BasicFile_$2 + $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicOperations tachyon://$MASTER_ADDRESS:19998 /BasicFile_$2 $2 exit 0 elif [[ "$1" == "BasicRawTable" ]]; then -- $bin/tachyon tfs rm /Basic_Raw_Table_$2 -+ $bin/tachyon.sh tfs rm /Basic_Raw_Table_$2 - $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicRawTableOperations $MASTER_ADDRESS:19998 /BasicRawTable_$2 $2 +- $bin/tachyon tfs rm /BasicRawTable_$2 ++ $bin/tachyon.sh tfs rm /BasicRawTable_$2 + $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicRawTableOperations tachyon://$MASTER_ADDRESS:19998 /BasicRawTable_$2 $2 exit 0 elif [[ "$1" == "BasicCheckpoint" ]]; then - $bin/tachyon tfs rm /BasicCheckpoint + $bin/tachyon.sh tfs rm /BasicCheckpoint - $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicCheckpoint $MASTER_ADDRESS:19998 /BasicCheckpoint 10 + $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicCheckpoint tachyon://$MASTER_ADDRESS:19998 /BasicCheckpoint 10 exit 0 fi -@@ -116,10 +116,10 @@ elif [ "$COMMAND" == "runTests" ]; then +@@ -126,10 +126,10 @@ elif [ "$COMMAND" == "runTests" ]; then for op in ${opArr[@]} do @@ -55,10 +55,10 @@ index d795ed0..7d1ce67 100755 exit 0 diff --git a/src/main/java/tachyon/conf/CommonConf.java b/src/main/java/tachyon/conf/CommonConf.java -index b3c6974..6e3af0a 100644 +index aad1b53..7359cbb 100644 --- a/src/main/java/tachyon/conf/CommonConf.java +++ b/src/main/java/tachyon/conf/CommonConf.java -@@ -42,14 +42,7 @@ public class CommonConf extends Utils { +@@ -43,14 +43,7 @@ public class CommonConf extends Utils { public final boolean ASYNC_ENABLED; private CommonConf() { @@ -71,26 +71,6 @@ index b3c6974..6e3af0a 100644 - } - TACHYON_HOME = getProperty("tachyon.home", "/mnt/tachyon_default_home"); + TACHYON_HOME = getProperty("tachyon.home", "/var/lib/tachyon"); + WEB_RESOURCES = getProperty("tachyon.web.resources", TACHYON_HOME + "/src/main/java/tachyon/web/resources"); UNDERFS_ADDRESS = getProperty("tachyon.underfs.address", TACHYON_HOME + "/underfs"); UNDERFS_DATA_FOLDER = getProperty("tachyon.data.folder", UNDERFS_ADDRESS + "/tachyon/data"); - UNDERFS_WORKERS_FOLDER = -diff --git a/src/main/java/tachyon/web/UIWebServer.java b/src/main/java/tachyon/web/UIWebServer.java -index 6d31515..b6f2c3b 100644 ---- a/src/main/java/tachyon/web/UIWebServer.java -+++ b/src/main/java/tachyon/web/UIWebServer.java -@@ -58,7 +58,7 @@ public class UIWebServer { - WebAppContext webappcontext = new WebAppContext(); - - webappcontext.setContextPath("/"); -- File warPath = new File(CommonConf.get().TACHYON_HOME + "/src/main/java/tachyon/web/resources"); -+ File warPath = new File("/usr/share/tachyon/web/resources"); - webappcontext.setWar(warPath.getAbsolutePath()); - HandlerList handlers = new HandlerList(); - webappcontext.addServlet( -@@ -90,4 +90,4 @@ public class UIWebServer { - public void shutdownWebServer() throws Exception { - mServer.stop(); - } --} -\ No newline at end of file -+} diff --git a/tachyon-0.4.1-permissions.patch b/tachyon-0.4.1-permissions.patch new file mode 100644 index 0000000..1bbb807 --- /dev/null +++ b/tachyon-0.4.1-permissions.patch @@ -0,0 +1,39 @@ +diff --git a/src/main/java/tachyon/worker/WorkerStorage.java b/src/main/java/tachyon/worker/WorkerStorage.java +index 592ddf8..95bf1ce 100644 +--- a/src/main/java/tachyon/worker/WorkerStorage.java ++++ b/src/main/java/tachyon/worker/WorkerStorage.java +@@ -25,6 +25,9 @@ import java.io.RandomAccessFile; + import java.net.InetSocketAddress; + import java.nio.ByteBuffer; + import java.nio.channels.FileChannel.MapMode; ++import java.nio.file.Files; ++import java.nio.file.attribute.PosixFilePermission; ++import java.nio.file.attribute.PosixFilePermissions; + import java.util.ArrayList; + import java.util.HashMap; + import java.util.HashSet; +@@ -513,10 +516,16 @@ public class WorkerStorage { + private void initializeWorkerStorage() throws IOException, FileDoesNotExistException, + SuspectedFileSizeException, BlockInfoException, TException { + LOG.info("Initializing the worker storage."); ++ ++ // Set Default directory permissions to 775 ++ Set perms = PosixFilePermissions.fromString("rwxrwxr-x"); ++ + if (!mLocalDataFolder.exists()) { + LOG.info("Local folder " + mLocalDataFolder + " does not exist. Creating a new one."); + mLocalDataFolder.mkdir(); + mLocalUserFolder.mkdir(); ++ Files.setPosixFilePermissions(mLocalDataFolder.toPath(), perms); ++ Files.setPosixFilePermissions(mLocalUserFolder.toPath(), perms); + return; + } + +@@ -534,6 +543,7 @@ public class WorkerStorage { + } + } + mLocalUserFolder.mkdir(); ++ Files.setPosixFilePermissions(mLocalUserFolder.toPath(), perms); + + mUnderfsOrphansFolder = mUnderfsWorkerFolder + "/orphans"; + if (!mUnderFs.exists(mUnderfsOrphansFolder)) { diff --git a/tachyon-env.sh b/tachyon-env.sh index 78e7e50..257008f 100644 --- a/tachyon-env.sh +++ b/tachyon-env.sh @@ -24,6 +24,7 @@ CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export TACHYON_JAVA_OPTS+=" -Dlog4j.configuration=file:$CONF_DIR/log4j.properties -Dhadoop.home.dir=/usr + -Dtachyon.web.resources=/usr/share/tachyon/web/resources -Dtachyon.debug=false -Dtachyon.underfs.address=$TACHYON_UNDERFS_ADDRESS -Dtachyon.worker.memory.size=$TACHYON_WORKER_MEMORY_SIZE diff --git a/tachyon-master.service b/tachyon-master.service index 2809b2a..02d61f9 100644 --- a/tachyon-master.service +++ b/tachyon-master.service @@ -5,17 +5,33 @@ After=network.target Wants=network.target [Service] -#EnvironmentFile=-/etc/sysconfig/tachyon-master Type=forking ExecStart=/usr/bin/tachyon-start.sh master ExecStop=/usr/bin/tachyon-killall.sh tachyon.Master User=hdfs -Group=hdfs +Group=hadoop LimitNOFILE=16384 Restart=always RestartSec=20 + #PIDFile=/var/run/tachyon/tachyon-master.pid -#ControlGroup=cpu:/tachyon +####################################### +# Note: Below are cgroup options +####################################### +#Slice= +#CPUAccounting=true +#CPUShares=1024 + +#MemoryAccounting=true +#TBD: MemoryLimit=bytes, MemorySoftLimit=bytes + +#BlockIOAccounting=true +#BlockIOWeight=?? +#BlockIODeviceWeight=?? +#TBD: BlockIOReadBandwidth=bytes, BlockIOWriteBandwidth=bytes + +#DeviceAllow= +#DevicePolicy=auto|closed|strict [Install] WantedBy=multi-user.target diff --git a/tachyon-slave.service b/tachyon-slave.service index 9b678cd..583f6e0 100644 --- a/tachyon-slave.service +++ b/tachyon-slave.service @@ -5,18 +5,34 @@ After=network.target Wants=network.target [Service] -#EnvironmentFile=-/etc/sysconfig/tachyon-slave Type=forking ExecStart=/usr/bin/tachyon-start.sh worker NoMount ExecStop=/usr/bin/tachyon-killall.sh tachyon.Worker ExecReload=/bin/kill -HUP $MAINPID User=hdfs -Group=hdfs +Group=hadoop LimitNOFILE=16384 Restart=always RestartSec=20 + #PIDFile=/var/run/tachyon/tachyon-slave.pid -#ControlGroup=cpu:/tachyon +####################################### +# Note: Below are cgroup options +####################################### +#Slice= +#CPUAccounting=true +#CPUShares=1024 + +#MemoryAccounting=true +#TBD: MemoryLimit=bytes, MemorySoftLimit=bytes + +#BlockIOAccounting=true +#BlockIOWeight=?? +#BlockIODeviceWeight=?? +#TBD: BlockIOReadBandwidth=bytes, BlockIOWriteBandwidth=bytes + +#DeviceAllow= +#DevicePolicy=auto|closed|strict [Install] WantedBy=multi-user.target