diff --git a/accumulo.spec b/accumulo.spec index 550b193..13648ca 100644 --- a/accumulo.spec +++ b/accumulo.spec @@ -9,7 +9,7 @@ Name: %{proj} Version: 1.6.6 -Release: 7%{?dist} +Release: 8%{?dist} Summary: A software platform for processing vast amounts of data License: ASL 2.0 Group: Development/Libraries @@ -46,11 +46,9 @@ Patch5: default-conf.patch %if 0%{?fedora} > 24 # Patch upstream ACCUMULO-3470 (update to commons-vfs 2.1) Patch6: ACCUMULO-3470.patch -# disable missing vfs2 provider for HDFS (bz#1387110) -Patch7: disable-hdfs-vfs2-provider.patch %endif # Fix for Shell erroneously reading accumulo-site.xml -Patch8: shell-not-read-conf.patch +Patch7: shell-not-read-conf.patch BuildRequires: apache-commons-cli BuildRequires: apache-commons-codec @@ -60,7 +58,11 @@ BuildRequires: apache-commons-io BuildRequires: apache-commons-lang BuildRequires: apache-commons-logging BuildRequires: apache-commons-math -BuildRequires: apache-commons-vfs +%if 0%{?fedora} > 24 +BuildRequires: apache-commons-vfs >= 2.1-7 +%else +BuildRequires: apache-commons-vfs < 2.1 +%endif BuildRequires: beust-jcommander BuildRequires: bouncycastle BuildRequires: exec-maven-plugin @@ -531,6 +533,9 @@ getent passwd %{name} >/dev/null || /usr/sbin/useradd --comment "%{longproj}" -- %endif %changelog +* Thu Nov 03 2016 Christopher Tubbs - 1.6.6-8 +- Re-enable VFS 2.1 HDFS Provider (bz#1387110) + * Wed Nov 02 2016 Mike Miller - 1.6.6-7 - Fix for Shell erroneously reading accumulo-site.xml diff --git a/disable-hdfs-vfs2-provider.patch b/disable-hdfs-vfs2-provider.patch deleted file mode 100644 index a22c0d9..0000000 --- a/disable-hdfs-vfs2-provider.patch +++ /dev/null @@ -1,334 +0,0 @@ -From eab2d704a122a4e9c3b278dd8fba8fc7e013e3d4 Mon Sep 17 00:00:00 2001 -From: Christopher Tubbs -Date: Sat, 22 Oct 2016 11:42:58 -0400 -Subject: [PATCH 1/1] Disable vfs2 provider for HDFS - ---- - .../classloader/vfs/AccumuloVFSClassLoader.java | 20 -- - .../providers/ReadOnlyHdfsFileProviderTest.java | 248 --------------------- - .../org/apache/accumulo/test/AccumuloDFSBase.java | 2 - - 3 files changed, 270 deletions(-) - delete mode 100644 start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java - -diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java -index 98b3b54..cb9fb19 100644 ---- a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java -+++ b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java -@@ -38,10 +38,6 @@ import org.apache.commons.vfs2.impl.DefaultFileSystemManager; - import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; - import org.apache.commons.vfs2.impl.VFSClassLoader; - import org.apache.commons.vfs2.provider.FileReplicator; --import org.apache.commons.vfs2.provider.hdfs.HdfsFileObject; --import org.apache.commons.vfs2.provider.hdfs.HdfsFileProvider; --import org.apache.hadoop.conf.Configuration; --import org.apache.hadoop.fs.FileSystem; - import org.apache.log4j.Logger; - - /** -@@ -219,21 +215,6 @@ public class AccumuloVFSClassLoader { - localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent)); - loader = localLoader; - -- // An HDFS FileSystem and Configuration object were created for each unique HDFS namespace in the call to resolve above. -- // The HDFS Client did us a favor and cached these objects so that the next time someone calls FileSystem.get(uri), they -- // get the cached object. However, these objects were created not with the system VFS classloader, but the classloader above -- // it. We need to override the classloader on the Configuration objects. Ran into an issue were log recovery was being attempted -- // and SequenceFile$Reader was trying to instantiate the key class via WritableName.getClass(String, Configuration) -- for (FileObject fo : vfsCP) { -- if (fo instanceof HdfsFileObject) { -- String uri = fo.getName().getRootURI(); -- Configuration c = new Configuration(true); -- c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri); -- FileSystem fs = FileSystem.get(c); -- fs.getConf().setClassLoader(loader.getClassLoader()); -- } -- } -- - } - } - } -@@ -263,7 +244,6 @@ public class AccumuloVFSClassLoader { - vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); - vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); - vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider()); -- vfs.addProvider("hdfs", new HdfsFileProvider()); - vfs.addExtensionMap("jar", "jar"); - vfs.addExtensionMap("zip", "zip"); - vfs.addExtensionMap("gz", "gz"); -diff --git a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java -deleted file mode 100644 -index 5b69df9..0000000 ---- a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java -+++ /dev/null -@@ -1,248 +0,0 @@ --/* -- * Licensed to the Apache Software Foundation (ASF) under one or more -- * contributor license agreements. See the NOTICE file distributed with -- * this work for additional information regarding copyright ownership. -- * The ASF licenses this file to You under the Apache License, Version 2.0 -- * (the "License"); you may not use this file except in compliance with -- * the License. You may obtain a copy of the License at -- * -- * http://www.apache.org/licenses/LICENSE-2.0 -- * -- * Unless required by applicable law or agreed to in writing, software -- * distributed under the License is distributed on an "AS IS" BASIS, -- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -- * See the License for the specific language governing permissions and -- * limitations under the License. -- */ --package org.apache.accumulo.start.classloader.vfs.providers; -- --import java.io.IOException; --import java.util.Map; -- --import org.apache.accumulo.test.AccumuloDFSBase; --import org.apache.commons.vfs2.FileObject; --import org.apache.commons.vfs2.FileSystemException; --import org.apache.commons.vfs2.FileType; --import org.apache.commons.vfs2.impl.DefaultFileSystemManager; --import org.apache.commons.vfs2.provider.hdfs.HdfsFileAttributes; --import org.apache.commons.vfs2.provider.hdfs.HdfsFileProvider; --import org.apache.commons.vfs2.util.RandomAccessMode; --import org.apache.hadoop.fs.FileSystem; --import org.apache.hadoop.fs.Path; --import org.junit.After; --import org.junit.Assert; --import org.junit.Before; --import org.junit.Test; -- --public class ReadOnlyHdfsFileProviderTest extends AccumuloDFSBase { -- -- private static final String TEST_DIR1 = getHdfsUri() + "/test-dir"; -- private static final Path DIR1_PATH = new Path("/test-dir"); -- private static final String TEST_FILE1 = TEST_DIR1 + "/accumulo-test-1.jar"; -- private static final Path FILE1_PATH = new Path(DIR1_PATH, "accumulo-test-1.jar"); -- -- private DefaultFileSystemManager manager = null; -- private FileSystem hdfs = null; -- -- @Before -- public void setup() throws Exception { -- manager = new DefaultFileSystemManager(); -- manager.addProvider("hdfs", new HdfsFileProvider()); -- manager.init(); -- this.hdfs = cluster.getFileSystem(); -- } -- -- private FileObject createTestFile(FileSystem hdfs) throws IOException { -- // Create the directory -- hdfs.mkdirs(DIR1_PATH); -- FileObject dir = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(dir); -- Assert.assertTrue(dir.exists()); -- Assert.assertTrue(dir.getType().equals(FileType.FOLDER)); -- -- // Create the file in the directory -- hdfs.create(FILE1_PATH).close(); -- FileObject f = manager.resolveFile(TEST_FILE1); -- Assert.assertNotNull(f); -- Assert.assertTrue(f.exists()); -- Assert.assertTrue(f.getType().equals(FileType.FILE)); -- return f; -- } -- -- @Test -- public void testInit() throws Exception { -- FileObject fo = manager.resolveFile(TEST_FILE1); -- Assert.assertNotNull(fo); -- } -- -- @Test -- public void testExistsFails() throws Exception { -- FileObject fo = manager.resolveFile(TEST_FILE1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- } -- -- @Test -- public void testExistsSucceeds() throws Exception { -- FileObject fo = manager.resolveFile(TEST_FILE1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the file -- @SuppressWarnings("unused") -- FileObject f = createTestFile(hdfs); -- -- } -- -- @Test(expected = UnsupportedOperationException.class) -- public void testCanRenameTo() throws Exception { -- FileObject fo = createTestFile(this.hdfs); -- Assert.assertNotNull(fo); -- fo.canRenameTo(fo); -- } -- -- @Test -- public void testDoListChildren() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- FileObject dir = file.getParent(); -- -- FileObject[] children = dir.getChildren(); -- Assert.assertTrue(children.length == 1); -- Assert.assertTrue(children[0].getName().equals(file.getName())); -- -- } -- -- @Test -- public void testGetContentSize() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- Assert.assertEquals(0, file.getContent().getSize()); -- } -- -- @Test -- public void testGetInputStream() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- file.getContent().getInputStream().close(); -- } -- -- @Test -- public void testIsHidden() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- Assert.assertFalse(file.isHidden()); -- } -- -- @Test -- public void testIsReadable() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- Assert.assertTrue(file.isReadable()); -- } -- -- @Test -- public void testIsWritable() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- Assert.assertFalse(file.isWriteable()); -- } -- -- @Test -- public void testLastModificationTime() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- Assert.assertFalse(-1 == file.getContent().getLastModifiedTime()); -- } -- -- @Test -- public void testGetAttributes() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- Map attributes = file.getContent().getAttributes(); -- Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.BLOCK_SIZE.toString())); -- Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.GROUP.toString())); -- Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.LAST_ACCESS_TIME.toString())); -- Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.LENGTH.toString())); -- Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.MODIFICATION_TIME.toString())); -- Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.OWNER.toString())); -- Assert.assertTrue(attributes.containsKey(HdfsFileAttributes.PERMISSIONS.toString())); -- } -- -- @Test(expected = FileSystemException.class) -- public void testRandomAccessContent() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- file.getContent().getRandomAccessContent(RandomAccessMode.READWRITE).close(); -- } -- -- @Test -- public void testRandomAccessContent2() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- file.getContent().getRandomAccessContent(RandomAccessMode.READ).close(); -- } -- -- @Test -- public void testEquals() throws Exception { -- FileObject fo = manager.resolveFile(TEST_DIR1); -- Assert.assertNotNull(fo); -- Assert.assertFalse(fo.exists()); -- -- // Create the test file -- FileObject file = createTestFile(hdfs); -- // Get a handle to the same file -- FileObject file2 = manager.resolveFile(TEST_FILE1); -- Assert.assertEquals(file, file2); -- } -- -- @After -- public void tearDown() throws Exception { -- if (null != hdfs) { -- hdfs.delete(DIR1_PATH, true); -- hdfs.close(); -- } -- manager.close(); -- } -- --} -diff --git a/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java b/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java -index 343a3ee..4214ce5 100644 ---- a/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java -+++ b/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java -@@ -28,7 +28,6 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache; - import org.apache.commons.vfs2.impl.DefaultFileReplicator; - import org.apache.commons.vfs2.impl.DefaultFileSystemManager; - import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; --import org.apache.commons.vfs2.provider.hdfs.HdfsFileProvider; - import org.apache.hadoop.conf.Configuration; - import org.apache.hadoop.hdfs.DFSConfigKeys; - import org.apache.hadoop.hdfs.MiniDFSCluster; -@@ -98,7 +97,6 @@ public class AccumuloDFSBase { - vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); - vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); - vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider()); -- vfs.addProvider("hdfs", new HdfsFileProvider()); - vfs.addExtensionMap("jar", "jar"); - vfs.addExtensionMap("zip", "zip"); - vfs.addExtensionMap("gz", "gz"); --- -2.7.4 -