|
William Benton |
bfe1e18 |
From 0b4c1430341d2f0d44f1339d4f8c7972af1d2821 Mon Sep 17 00:00:00 2001
|
|
William Benton |
ed1b62c |
From: William Benton <willb@redhat.com>
|
|
William Benton |
ed1b62c |
Date: Sun, 23 Feb 2014 17:22:07 -0600
|
|
William Benton |
ed1b62c |
Subject: [PATCH 1/9] Replace lift-json with json4s-jackson.
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
The aim of the Json4s project is to provide a common API for
|
|
William Benton |
ed1b62c |
Scala JSON libraries. It is Apache-licensed, easier for
|
|
William Benton |
ed1b62c |
downstream distributions to package, and mostly API-compatible
|
|
William Benton |
ed1b62c |
with lift-json. Furthermore, the Jackson-backed implementation
|
|
William Benton |
ed1b62c |
parses faster than lift-json on all but the smallest inputs.
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
Backported patch from master to 0.9.0.
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
Conflicts:
|
|
William Benton |
ed1b62c |
core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
|
|
William Benton |
ed1b62c |
core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala
|
|
William Benton |
ed1b62c |
core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
|
|
William Benton |
ed1b62c |
core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
|
|
William Benton |
ed1b62c |
---
|
|
William Benton |
ed1b62c |
core/pom.xml | 5 +++--
|
|
William Benton |
ed1b62c |
.../apache/spark/deploy/FaultToleranceTest.scala | 9 ++++----
|
|
William Benton |
ed1b62c |
.../org/apache/spark/deploy/JsonProtocol.scala | 2 +-
|
|
William Benton |
ed1b62c |
.../spark/deploy/master/ui/ApplicationPage.scala | 2 +-
|
|
William Benton |
ed1b62c |
.../apache/spark/deploy/master/ui/IndexPage.scala | 2 +-
|
|
William Benton |
ed1b62c |
.../apache/spark/deploy/worker/ui/IndexPage.scala | 2 +-
|
|
William Benton |
ed1b62c |
.../scala/org/apache/spark/ui/JettyUtils.scala | 8 ++++----
|
|
William Benton |
ed1b62c |
.../apache/spark/deploy/JsonProtocolSuite.scala | 24 ++++++++++++++++++----
|
|
William Benton |
ed1b62c |
project/SparkBuild.scala | 2 +-
|
|
William Benton |
ed1b62c |
9 files changed, 37 insertions(+), 19 deletions(-)
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
diff --git a/core/pom.xml b/core/pom.xml
|
|
William Benton |
f4700ea |
index 8111400..e7aa19a 100644
|
|
William Benton |
ed1b62c |
--- a/core/pom.xml
|
|
William Benton |
ed1b62c |
+++ b/core/pom.xml
|
|
William Benton |
f4700ea |
@@ -104,8 +104,9 @@
|
|
William Benton |
ed1b62c |
<artifactId>scala-library</artifactId>
|
|
William Benton |
ed1b62c |
</dependency>
|
|
William Benton |
ed1b62c |
<dependency>
|
|
William Benton |
ed1b62c |
- <groupId>net.liftweb</groupId>
|
|
William Benton |
ed1b62c |
- <artifactId>lift-json_${scala.binary.version}</artifactId>
|
|
William Benton |
ed1b62c |
+ <groupId>org.json4s</groupId>
|
|
William Benton |
ed1b62c |
+ <artifactId>json4s-jackson_${scala.binary.version}</artifactId>
|
|
William Benton |
ed1b62c |
+ <version>3.2.6</version>
|
|
William Benton |
ed1b62c |
</dependency>
|
|
William Benton |
ed1b62c |
<dependency>
|
|
William Benton |
ed1b62c |
<groupId>it.unimi.dsi</groupId>
|
|
William Benton |
ed1b62c |
diff --git a/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala b/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
|
|
William Benton |
ed1b62c |
index 4dfb19e..60a87af 100644
|
|
William Benton |
ed1b62c |
--- a/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
|
|
William Benton |
ed1b62c |
+++ b/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
|
|
William Benton |
ed1b62c |
@@ -29,7 +29,8 @@ import scala.concurrent.ExecutionContext.Implicits.global
|
|
William Benton |
ed1b62c |
import scala.collection.mutable.ListBuffer
|
|
William Benton |
ed1b62c |
import scala.sys.process._
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
-import net.liftweb.json.JsonParser
|
|
William Benton |
ed1b62c |
+import org.json4s._
|
|
William Benton |
ed1b62c |
+import org.json4s.jackson.JsonMethods
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import org.apache.spark.{Logging, SparkContext}
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.master.RecoveryState
|
|
William Benton |
ed1b62c |
@@ -312,7 +313,7 @@ private[spark] object FaultToleranceTest extends App with Logging {
|
|
William Benton |
ed1b62c |
private[spark] class TestMasterInfo(val ip: String, val dockerId: DockerId, val logFile: File)
|
|
William Benton |
ed1b62c |
extends Logging {
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
- implicit val formats = net.liftweb.json.DefaultFormats
|
|
William Benton |
ed1b62c |
+ implicit val formats = org.json4s.DefaultFormats
|
|
William Benton |
ed1b62c |
var state: RecoveryState.Value = _
|
|
William Benton |
ed1b62c |
var liveWorkerIPs: List[String] = _
|
|
William Benton |
ed1b62c |
var numLiveApps = 0
|
|
William Benton |
ed1b62c |
@@ -322,7 +323,7 @@ private[spark] class TestMasterInfo(val ip: String, val dockerId: DockerId, val
|
|
William Benton |
ed1b62c |
def readState() {
|
|
William Benton |
ed1b62c |
try {
|
|
William Benton |
ed1b62c |
val masterStream = new InputStreamReader(new URL("http://%s:8080/json".format(ip)).openStream)
|
|
William Benton |
ed1b62c |
- val json = JsonParser.parse(masterStream, closeAutomatically = true)
|
|
William Benton |
ed1b62c |
+ val json = JsonMethods.parse(masterStream)
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
val workers = json \ "workers"
|
|
William Benton |
ed1b62c |
val liveWorkers = workers.children.filter(w => (w \ "state").extract[String] == "ALIVE")
|
|
William Benton |
ed1b62c |
@@ -350,7 +351,7 @@ private[spark] class TestMasterInfo(val ip: String, val dockerId: DockerId, val
|
|
William Benton |
ed1b62c |
private[spark] class TestWorkerInfo(val ip: String, val dockerId: DockerId, val logFile: File)
|
|
William Benton |
ed1b62c |
extends Logging {
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
- implicit val formats = net.liftweb.json.DefaultFormats
|
|
William Benton |
ed1b62c |
+ implicit val formats = org.json4s.DefaultFormats
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
logDebug("Created worker: " + this)
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
diff --git a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
|
|
William Benton |
ed1b62c |
index e607b8c..a43d004 100644
|
|
William Benton |
ed1b62c |
--- a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
|
|
William Benton |
ed1b62c |
+++ b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
|
|
William Benton |
ed1b62c |
@@ -17,7 +17,7 @@
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
package org.apache.spark.deploy
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
-import net.liftweb.json.JsonDSL._
|
|
William Benton |
ed1b62c |
+import org.json4s.JsonDSL._
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.master.{ApplicationInfo, WorkerInfo}
|
|
William Benton |
ed1b62c |
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
|
|
William Benton |
ed1b62c |
index f29a6ad..cba89dc 100644
|
|
William Benton |
ed1b62c |
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
|
|
William Benton |
ed1b62c |
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
|
|
William Benton |
ed1b62c |
@@ -22,7 +22,7 @@ import scala.xml.Node
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import akka.pattern.ask
|
|
William Benton |
ed1b62c |
import javax.servlet.http.HttpServletRequest
|
|
William Benton |
ed1b62c |
-import net.liftweb.json.JsonAST.JValue
|
|
William Benton |
ed1b62c |
+import org.json4s.JValue
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.JsonProtocol
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState}
|
|
William Benton |
ed1b62c |
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala
|
|
William Benton |
ed1b62c |
index b549825..aa8beff 100644
|
|
William Benton |
ed1b62c |
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala
|
|
William Benton |
ed1b62c |
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala
|
|
William Benton |
ed1b62c |
@@ -23,7 +23,7 @@ import scala.xml.Node
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import akka.pattern.ask
|
|
William Benton |
ed1b62c |
import javax.servlet.http.HttpServletRequest
|
|
William Benton |
ed1b62c |
-import net.liftweb.json.JsonAST.JValue
|
|
William Benton |
ed1b62c |
+import org.json4s.JValue
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.JsonProtocol
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState}
|
|
William Benton |
ed1b62c |
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala
|
|
William Benton |
ed1b62c |
index 925c6fb..de356dc 100644
|
|
William Benton |
ed1b62c |
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala
|
|
William Benton |
ed1b62c |
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala
|
|
William Benton |
ed1b62c |
@@ -22,7 +22,7 @@ import scala.xml.Node
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import akka.pattern.ask
|
|
William Benton |
ed1b62c |
import javax.servlet.http.HttpServletRequest
|
|
William Benton |
ed1b62c |
-import net.liftweb.json.JsonAST.JValue
|
|
William Benton |
ed1b62c |
+import org.json4s.JValue
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.JsonProtocol
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.DeployMessages.{RequestWorkerState, WorkerStateResponse}
|
|
William Benton |
ed1b62c |
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
|
|
William Benton |
ed1b62c |
index 7211dbc..4e43fd5 100644
|
|
William Benton |
ed1b62c |
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
|
|
William Benton |
ed1b62c |
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
|
|
William Benton |
ed1b62c |
@@ -23,10 +23,10 @@ import scala.annotation.tailrec
|
|
William Benton |
ed1b62c |
import scala.util.{Try, Success, Failure}
|
|
William Benton |
ed1b62c |
import scala.xml.Node
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
-import net.liftweb.json.{JValue, pretty, render}
|
|
William Benton |
ed1b62c |
-
|
|
William Benton |
ed1b62c |
-import org.eclipse.jetty.server.{Server, Request, Handler}
|
|
William Benton |
ed1b62c |
-import org.eclipse.jetty.server.handler.{ResourceHandler, HandlerList, ContextHandler, AbstractHandler}
|
|
William Benton |
ed1b62c |
+import org.json4s.JValue
|
|
William Benton |
ed1b62c |
+import org.json4s.jackson.JsonMethods.{pretty, render}
|
|
William Benton |
ed1b62c |
+import org.eclipse.jetty.server.{Handler, Request, Server}
|
|
William Benton |
ed1b62c |
+import org.eclipse.jetty.server.handler.{AbstractHandler, ContextHandler, HandlerList, ResourceHandler}
|
|
William Benton |
ed1b62c |
import org.eclipse.jetty.util.thread.QueuedThreadPool
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import org.apache.spark.Logging
|
|
William Benton |
ed1b62c |
diff --git a/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
|
|
William Benton |
ed1b62c |
index d05bbd6..8f1df8a 100644
|
|
William Benton |
ed1b62c |
--- a/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
|
|
William Benton |
ed1b62c |
+++ b/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
|
|
William Benton |
ed1b62c |
@@ -20,8 +20,12 @@ package org.apache.spark.deploy
|
|
William Benton |
ed1b62c |
import java.io.File
|
|
William Benton |
ed1b62c |
import java.util.Date
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
-import net.liftweb.json.{JsonAST, JsonParser}
|
|
William Benton |
ed1b62c |
-import net.liftweb.json.JsonAST.JValue
|
|
William Benton |
ed1b62c |
+import org.json4s._
|
|
William Benton |
ed1b62c |
+
|
|
William Benton |
ed1b62c |
+import org.json4s.JValue
|
|
William Benton |
ed1b62c |
+import org.json4s.jackson.JsonMethods
|
|
William Benton |
ed1b62c |
+import com.fasterxml.jackson.core.JsonParseException
|
|
William Benton |
ed1b62c |
+
|
|
William Benton |
ed1b62c |
import org.scalatest.FunSuite
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
|
|
William Benton |
ed1b62c |
@@ -32,21 +36,31 @@ class JsonProtocolSuite extends FunSuite {
|
|
William Benton |
ed1b62c |
test("writeApplicationInfo") {
|
|
William Benton |
ed1b62c |
val output = JsonProtocol.writeApplicationInfo(createAppInfo())
|
|
William Benton |
ed1b62c |
assertValidJson(output)
|
|
William Benton |
ed1b62c |
+ assertValidDataInJson(output, JsonMethods.parse(JsonConstants.appInfoJsonStr))
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
test("writeWorkerInfo") {
|
|
William Benton |
ed1b62c |
val output = JsonProtocol.writeWorkerInfo(createWorkerInfo())
|
|
William Benton |
ed1b62c |
assertValidJson(output)
|
|
William Benton |
ed1b62c |
+ assertValidDataInJson(output, JsonMethods.parse(JsonConstants.workerInfoJsonStr))
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
test("writeApplicationDescription") {
|
|
William Benton |
ed1b62c |
val output = JsonProtocol.writeApplicationDescription(createAppDesc())
|
|
William Benton |
ed1b62c |
assertValidJson(output)
|
|
William Benton |
ed1b62c |
+ assertValidDataInJson(output, JsonMethods.parse(JsonConstants.appDescJsonStr))
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
test("writeExecutorRunner") {
|
|
William Benton |
ed1b62c |
val output = JsonProtocol.writeExecutorRunner(createExecutorRunner())
|
|
William Benton |
ed1b62c |
assertValidJson(output)
|
|
William Benton |
ed1b62c |
+ assertValidDataInJson(output, JsonMethods.parse(JsonConstants.executorRunnerJsonStr))
|
|
William Benton |
ed1b62c |
+ }
|
|
William Benton |
ed1b62c |
+
|
|
William Benton |
ed1b62c |
+ test("writeDriverInfo") {
|
|
William Benton |
ed1b62c |
+ val output = JsonProtocol.writeDriverInfo(createDriverInfo())
|
|
William Benton |
ed1b62c |
+ assertValidJson(output)
|
|
William Benton |
ed1b62c |
+ assertValidDataInJson(output, JsonMethods.parse(JsonConstants.driverInfoJsonStr))
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
test("writeMasterState") {
|
|
William Benton |
ed1b62c |
@@ -59,6 +73,7 @@ class JsonProtocolSuite extends FunSuite {
|
|
William Benton |
ed1b62c |
activeDrivers, completedDrivers, RecoveryState.ALIVE)
|
|
William Benton |
ed1b62c |
val output = JsonProtocol.writeMasterState(stateResponse)
|
|
William Benton |
ed1b62c |
assertValidJson(output)
|
|
William Benton |
ed1b62c |
+ assertValidDataInJson(output, JsonMethods.parse(JsonConstants.masterStateJsonStr))
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
test("writeWorkerState") {
|
|
William Benton |
ed1b62c |
@@ -70,6 +85,7 @@ class JsonProtocolSuite extends FunSuite {
|
|
William Benton |
ed1b62c |
finishedExecutors, drivers, finishedDrivers, "masterUrl", 4, 1234, 4, 1234, "masterWebUiUrl")
|
|
William Benton |
ed1b62c |
val output = JsonProtocol.writeWorkerState(stateResponse)
|
|
William Benton |
ed1b62c |
assertValidJson(output)
|
|
William Benton |
ed1b62c |
+ assertValidDataInJson(output, JsonMethods.parse(JsonConstants.workerStateJsonStr))
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
def createAppDesc(): ApplicationDescription = {
|
|
William Benton |
ed1b62c |
@@ -106,9 +122,9 @@ class JsonProtocolSuite extends FunSuite {
|
|
William Benton |
ed1b62c |
|
|
William Benton |
ed1b62c |
def assertValidJson(json: JValue) {
|
|
William Benton |
ed1b62c |
try {
|
|
William Benton |
ed1b62c |
- JsonParser.parse(JsonAST.compactRender(json))
|
|
William Benton |
ed1b62c |
+ JsonMethods.parse(JsonMethods.compact(json))
|
|
William Benton |
ed1b62c |
} catch {
|
|
William Benton |
ed1b62c |
- case e: JsonParser.ParseException => fail("Invalid Json detected", e)
|
|
William Benton |
ed1b62c |
+ case e: JsonParseException => fail("Invalid Json detected", e)
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
}
|
|
William Benton |
ed1b62c |
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
|
|
William Benton |
f4700ea |
index 52e894e..6346b29 100644
|
|
William Benton |
ed1b62c |
--- a/project/SparkBuild.scala
|
|
William Benton |
ed1b62c |
+++ b/project/SparkBuild.scala
|
|
William Benton |
f4700ea |
@@ -274,7 +274,7 @@ object SparkBuild extends Build {
|
|
William Benton |
ed1b62c |
"org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
|
|
William Benton |
ed1b62c |
"org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
|
|
William Benton |
ed1b62c |
"org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test",
|
|
William Benton |
ed1b62c |
- "net.liftweb" %% "lift-json" % "2.5.1" excludeAll(excludeNetty),
|
|
William Benton |
ed1b62c |
+ "org.json4s" %% "json4s-jackson" % "3.2.6",
|
|
William Benton |
ed1b62c |
"it.unimi.dsi" % "fastutil" % "6.4.4",
|
|
William Benton |
ed1b62c |
"colt" % "colt" % "1.2.0",
|
|
William Benton |
ed1b62c |
"org.apache.mesos" % "mesos" % "0.13.0",
|
|
William Benton |
ed1b62c |
--
|
|
William Benton |
ed1b62c |
1.8.3.4 (Apple Git-47)
|
|
William Benton |
ed1b62c |
|