Replace json4s with com.lihaoyi.upickle / ujson (#6331)

Also bumped cpg to get rid of all json4s artifacts in the whole dependency tree.

For: https://harness.atlassian.net/browse/QT-1539

Author: max-leuthaeuser

MODULE.bazel

@@ -3,7 +3,7 @@ module(name = "joern")
 bazel_dep(name = "rules_scala", version = "7.2.5")
 bazel_dep(name = "codepropertygraph")
 
-CODEPROPERTYGRAPH_VERSION = "9d4b36e50cbffefcf9730e3afa9d4522e4f19e7c"
+CODEPROPERTYGRAPH_VERSION = "1735c289d7258d4ef02f6c39ce635581cca4f330"
 
 BAZEL_TOOLING_VERSION = "fbdfc66995360c408ecf94d34837a0355aa7ac4d"
 
@@ -49,9 +49,8 @@ maven.install(
         "org.projectlombok:lombok:1.18.42",
         "net.lingala.zip4j:zip4j:2.11.5",
         "org.gradle:gradle-tooling-api:8.3",
-        "com.lihaoyi:ujson_3:4.0.2",
-        "com.lihaoyi:upickle_3:4.0.2",
-        "org.json4s:json4s-native_3:4.0.7",
+        "com.lihaoyi:ujson_3:4.4.3",
+        "com.lihaoyi:upickle_3:4.4.3",
         "com.michaelpollmeier:scala-repl-pp_3.3.6:0.6.3",
         "commons-io:commons-io:2.22.0",
         "org.apache.commons:commons-lang3:3.14.0",

bazel/reexports/BUILD

@@ -16,8 +16,11 @@ scala_library(
     name = "com_lihaoyi_upickle_with_deps",
     visibility = ["//visibility:public"],
     exports = [
+        "@maven//:com_lihaoyi_geny_3",
         "@maven//:com_lihaoyi_ujson_3",
         "@maven//:com_lihaoyi_upickle_3",
+        "@maven//:com_lihaoyi_upickle_core_3",
+        "@maven//:com_lihaoyi_upickle_implicits_3",
     ],
     deps = [
     ],

build.sbt

@@ -2,7 +2,7 @@ name                     := "joern"
 ThisBuild / organization := "io.joern"
 ThisBuild / scalaVersion := "3.8.3"
 
-val cpgVersion = "1.7.77"
+val cpgVersion = "1.7.78"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb

console/build.sbt

@@ -18,6 +18,7 @@ libraryDependencies ++= Seq(
   "org.zeroturnaround"    % "zt-zip"               % Versions.zeroTurnaround,
   "com.lihaoyi"          %% "os-lib"               % Versions.osLib,
   "com.lihaoyi"          %% "pprint"               % Versions.pPrint,
+  "com.lihaoyi"          %% "upickle"              % Versions.upickle,
   "com.lihaoyi"          %% "cask"                 % Versions.cask,
   "org.apache.commons"    % "commons-lang3"        % Versions.commonsLang,
   "org.scalatest"        %% "scalatest"            % Versions.scalatest % Test

console/src/main/scala/io/joern/console/workspacehandling/Project.scala

@@ -14,6 +14,10 @@ object Project {
 
 case class ProjectFile(inputPath: String, name: String)
 
+object ProjectFile {
+  implicit val readWriter: upickle.default.ReadWriter[ProjectFile] = upickle.default.macroRW
+}
+
 /** @param path
   *   absolute path to directory holding the project
   * @param cpg

console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala

@@ -7,8 +7,6 @@ import io.shiftleft.semanticcpg.utils.FileUtil.*
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
 import io.shiftleft.semanticcpg.utils.FileUtil
-import org.json4s.DefaultFormats
-import org.json4s.native.Serialization.write as jsonWrite
 
 import java.net.URLEncoder
 import java.nio.charset.Charset
@@ -86,16 +84,10 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo
 
   /** Write the project's `project.json`, a JSON file that holds meta information.
     */
-  private def writeProjectFile(projectFile: ProjectFile, dirPath: Path): Path = {
-    // TODO proguard and json4s don't play along. We actually want to
-    // serialize the case class ProjectFile here, but it comes out
-    // empty. This code will be moved to `codepropertgraph` at
-    // which point serialization should work.
-    // val content = jsonWrite(projectFile)
-    implicit val formats: DefaultFormats.type = DefaultFormats
-    val PROJECTFILE_NAME                      = "project.json"
-    val content     = jsonWrite(Map("inputPath" -> projectFile.inputPath, "name" -> projectFile.name))
-    val projectPath = dirPath.resolve(PROJECTFILE_NAME)
+  private[workspacehandling] def writeProjectFile(projectFile: ProjectFile, dirPath: Path): Path = {
+    val PROJECTFILE_NAME = "project.json"
+    val content          = upickle.default.write(projectFile)
+    val projectPath      = dirPath.resolve(PROJECTFILE_NAME)
     Files.writeString(projectPath, content)
     projectPath
   }

console/src/test/scala/io/joern/console/workspacehandling/WorkspaceLoaderTests.scala

@@ -60,12 +60,12 @@ class WorkspaceLoaderTests extends AnyWordSpec with Matchers {
 
   "ProjectFile" should {
 
-    import org.json4s.DefaultFormats
-    import org.json4s.native.Serialization.{read => jsonRead, write => jsonWrite}
-    implicit val formats: DefaultFormats.type = DefaultFormats
-
     "be serializable to json" in {
-      jsonWrite(ProjectFile("foo", "aname")) shouldBe """{"inputPath":"foo","name":"aname"}"""
+      FileUtil.usingTemporaryDirectory(tmpDirPrefix) { tmpDir =>
+        val manager     = new WorkspaceManager(tmpDir.toString)
+        val projectPath = manager.writeProjectFile(ProjectFile("foo", "aname"), tmpDir)
+        Files.readString(projectPath) shouldBe """{"inputPath":"foo","name":"aname"}"""
+      }
     }
 
   }

joern-cli/frontends/abap2cpg/build.sbt

@@ -11,7 +11,7 @@ dependsOn(
 
 libraryDependencies ++= Seq(
   "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
-  "com.lihaoyi"   %% "ujson"             % "4.1.0",
+  "com.lihaoyi"   %% "ujson"             % Versions.upickle,
   "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 

joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala

@@ -1,7 +1,7 @@
 package io.joern.joerncli
 
 import io.joern.console.scan.{ScanPass, outputFindings}
-import io.joern.console.{BridgeBase, DefaultArgumentProvider, Query, QueryDatabase}
+import io.joern.console.{BridgeBase, DefaultArgumentProvider, Query, QueryDatabase, QueryJsonSerialization}
 import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext}
 import io.joern.dataflowengineoss.semanticsloader.{NoSemantics, Semantics}
 import io.joern.joerncli.JoernScan.getQueriesFromQueryDb
@@ -12,8 +12,6 @@ import io.shiftleft.semanticcpg.language.locationCreator
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
 import io.shiftleft.semanticcpg.utils.FileUtil
 import io.shiftleft.semanticcpg.utils.FileUtil.*
-import org.json4s.native.Serialization
-import org.json4s.{Formats, NoTypeHints}
 import scopt.OptionParser
 
 import java.nio.file.{Files, Path, Paths}
@@ -134,10 +132,9 @@ object JoernScan extends BridgeBase {
 
   private def dumpQueriesAsJson(outFileName: String): Unit = {
     implicit val engineContext: EngineContext = EngineContext(NoSemantics)
-    implicit val formats: AnyRef & Formats    = Serialization.formats(NoTypeHints)
     val queryDb                               = new QueryDatabase(new JoernDefaultArgumentProvider(0))
     val outFile                               = Paths.get(outFileName)
-    Files.writeString(outFile, Serialization.write(queryDb.allQueries))
+    Files.writeString(outFile, QueryJsonSerialization.write(queryDb.allQueries))
     println(s"Queries written to: $outFileName")
   }
 

joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala

@@ -5,8 +5,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.PropertyNames
 import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Method}
 import io.shiftleft.semanticcpg.language.*
-import org.json4s.DefaultFormats
-import org.json4s.native.Serialization
+import io.shiftleft.semanticcpg.utils.JsonRenderer
 
 import scala.collection.mutable
 import scala.jdk.CollectionConverters.*
@@ -119,17 +118,16 @@ trait EmbeddingGenerator[T, S] {
 
   def objectToString(t: T): String
 
-  implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats
-
   def vectorToString(vector: Map[S, Double]): String = defaultToString(vector)
 
-  def defaultToString[M](v: M): String = Serialization.write(v)
+  def defaultToString[M](value: M): String = ujson.write(JoernVectors.jsonRenderer.render(value))
 
 }
 
 object JoernVectors {
 
-  implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats
+  private[joerncli] val jsonRenderer = new JsonRenderer()
+
   case class Config(cpgFileName: String = "cpg.bin", outDir: String = "out", dimToFeature: Boolean = false)
 
   def main(args: Array[String]) = {
@@ -143,7 +141,7 @@ object JoernVectors {
         traversalToJson(embedding.objects, generator.defaultToString)
         if (config.dimToFeature) {
           println(",\"dimToFeature\": ")
-          println(Serialization.write(embedding.dimToStructure))
+          println(generator.defaultToString(embedding.dimToStructure))
         }
         println(",\"vectors\":")
         traversalToJson(embedding.vectors, generator.vectorToString)