diff --git a/build.sbt b/build.sbt index 755831fe..6b553937 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings val akkaVersion = "2.6.20" val algebirdVersion = "0.13.9" val bijectionVersion = "0.9.7" -val kryoVersion = "4.0.2" +val kryoVersion = "5.5.0" val scroogeVersion = "21.2.0" val asmVersion = "4.16" val protobufVersion = "3.22.2" @@ -53,7 +53,7 @@ val sharedSettings = Seq( "org.scalacheck" %% "scalacheck" % "1.15.2" % "test", "org.scalatest" %% "scalatest" % "3.2.15" % "test", "org.scalatestplus" %% "scalatestplus-scalacheck" % "3.1.0.0-RC2" % "test", - "com.esotericsoftware" % "kryo-shaded" % kryoVersion + "com.esotericsoftware" % "kryo" % kryoVersion ), Test / parallelExecution := true, pomExtra := https://github.com/twitter/chill diff --git a/chill-akka/src/main/scala/com/twitter/chill/akka/ActorRefSerializer.scala b/chill-akka/src/main/scala/com/twitter/chill/akka/ActorRefSerializer.scala index 365a2ef8..66ee3836 100644 --- a/chill-akka/src/main/scala/com/twitter/chill/akka/ActorRefSerializer.scala +++ b/chill-akka/src/main/scala/com/twitter/chill/akka/ActorRefSerializer.scala @@ -34,7 +34,7 @@ class ActorRefSerializer(system: ExtendedActorSystem) extends Serializer[ActorRe kryo.forSubclass[ActorRef](this) } - override def read(kryo: Kryo, input: Input, typ: Class[ActorRef]): ActorRef = { + override def read(kryo: Kryo, input: Input, typ: Class[_ <: ActorRef]): ActorRef = { val path = ActorPath.fromString(input.readString()) system.provider.resolveActorRef(path) } diff --git a/chill-algebird/src/main/scala/com/twitter/chill/algebird/AlgebirdSerializers.scala b/chill-algebird/src/main/scala/com/twitter/chill/algebird/AlgebirdSerializers.scala index 59f13e93..f1c6ad61 100644 --- a/chill-algebird/src/main/scala/com/twitter/chill/algebird/AlgebirdSerializers.scala +++ b/chill-algebird/src/main/scala/com/twitter/chill/algebird/AlgebirdSerializers.scala @@ -29,7 +29,7 @@ class AveragedValueSerializer extends KSerializer[AveragedValue] { out.writeLong(s.count, true) out.writeDouble(s.value) } - def read(kser: Kryo, in: Input, cls: Class[AveragedValue]): AveragedValue = + def read(kser: Kryo, in: Input, cls: Class[_ <: AveragedValue]): AveragedValue = AveragedValue(in.readLong(true), in.readDouble) } @@ -42,7 +42,7 @@ class MomentsSerializer extends KSerializer[Moments] { out.writeDouble(s.m3) out.writeDouble(s.m4) } - def read(kser: Kryo, in: Input, cls: Class[Moments]): Moments = + def read(kser: Kryo, in: Input, cls: Class[_ <: Moments]): Moments = Moments(in.readLong(true), in.readDouble, in.readDouble, in.readDouble, in.readDouble) } @@ -52,7 +52,7 @@ class DecayedValueSerializer extends KSerializer[DecayedValue] { out.writeDouble(s.value) out.writeDouble(s.scaledTime) } - def read(kser: Kryo, in: Input, cls: Class[DecayedValue]): DecayedValue = + def read(kser: Kryo, in: Input, cls: Class[_ <: DecayedValue]): DecayedValue = DecayedValue(in.readDouble, in.readDouble) } @@ -63,7 +63,7 @@ class HLLSerializer extends KSerializer[HLL] { out.writeInt(bytes.size, true) out.writeBytes(bytes) } - def read(kser: Kryo, in: Input, cls: Class[HLL]): HLL = + def read(kser: Kryo, in: Input, cls: Class[_ <: HLL]): HLL = HyperLogLog.fromBytes(in.readBytes(in.readInt(true))) } @@ -72,7 +72,7 @@ class HLLMonoidSerializer extends KSerializer[HyperLogLogMonoid] { val hllMonoids: MMap[Int, HyperLogLogMonoid] = MMap[Int, HyperLogLogMonoid]() def write(kser: Kryo, out: Output, mon: HyperLogLogMonoid): Unit = out.writeInt(mon.bits, true) - def read(kser: Kryo, in: Input, cls: Class[HyperLogLogMonoid]): HyperLogLogMonoid = { + def read(kser: Kryo, in: Input, cls: Class[_ <: HyperLogLogMonoid]): HyperLogLogMonoid = { val bits = in.readInt(true) hllMonoids.getOrElseUpdate(bits, new HyperLogLogMonoid(bits)) } @@ -80,7 +80,7 @@ class HLLMonoidSerializer extends KSerializer[HyperLogLogMonoid] { class QTreeSerializer extends KSerializer[QTree[Any]] { setImmutable(true) - override def read(kryo: Kryo, input: Input, cls: Class[QTree[Any]]): QTree[Any] = { + override def read(kryo: Kryo, input: Input, cls: Class[_ <: QTree[Any]]): QTree[Any] = { val (v1, v2, v3) = (input.readLong(), input.readInt(), input.readLong()) val v4 = kryo.readClassAndObject(input) val v5 = kryo.readClassAndObject(input).asInstanceOf[Option[QTree[Any]]] diff --git a/chill-avro/src/test/java/com/twitter/chill/avro/AvroSerializerJavaTest.java b/chill-avro/src/test/java/com/twitter/chill/avro/AvroSerializerJavaTest.java index 921c44cc..53258547 100644 --- a/chill-avro/src/test/java/com/twitter/chill/avro/AvroSerializerJavaTest.java +++ b/chill-avro/src/test/java/com/twitter/chill/avro/AvroSerializerJavaTest.java @@ -3,6 +3,7 @@ import avro.FiscalRecord; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Serializer; +import org.objenesis.strategy.StdInstantiatorStrategy; import com.twitter.chill.KryoInstantiator; import com.twitter.chill.KryoPool; import org.apache.avro.Schema; @@ -11,7 +12,6 @@ import org.apache.avro.generic.GenericRecordBuilder; import org.junit.Before; import org.junit.Test; -import org.objenesis.strategy.StdInstantiatorStrategy; import scala.reflect.ClassTag; import static org.junit.Assert.assertEquals; diff --git a/chill-bijection/src/main/scala/com/twitter/chill/BijectionEnrichedKryo.scala b/chill-bijection/src/main/scala/com/twitter/chill/BijectionEnrichedKryo.scala index aafd195d..010deecf 100644 --- a/chill-bijection/src/main/scala/com/twitter/chill/BijectionEnrichedKryo.scala +++ b/chill-bijection/src/main/scala/com/twitter/chill/BijectionEnrichedKryo.scala @@ -32,7 +32,7 @@ object BijectionEnrichedKryo { )(implicit bij: ImplicitBijection[A, B], cmf: ClassTag[B]): KSerializer[A] = new KSerializer[A] { def write(k: Kryo, out: Output, obj: A): Unit = kser.write(k, out, bij(obj)) - def read(k: Kryo, in: Input, cls: Class[A]): A = + def read(k: Kryo, in: Input, cls: Class[_ <: A]): A = bij.invert(kser.read(k, in, cmf.runtimeClass.asInstanceOf[Class[B]])) } diff --git a/chill-bijection/src/main/scala/com/twitter/chill/InjectiveSerializer.scala b/chill-bijection/src/main/scala/com/twitter/chill/InjectiveSerializer.scala index fce22244..4d4db9d9 100644 --- a/chill-bijection/src/main/scala/com/twitter/chill/InjectiveSerializer.scala +++ b/chill-bijection/src/main/scala/com/twitter/chill/InjectiveSerializer.scala @@ -29,7 +29,7 @@ class InjectiveSerializer[T] private (injection: Injection[T, Array[Byte]]) out.writeBytes(bytes) } - def read(kser: Kryo, in: Input, cls: Class[T]): T = { + def read(kser: Kryo, in: Input, cls: Class[_ <: T]): T = { val bytes = new Array[Byte](in.readInt(true)) in.readBytes(bytes) injection.invert(bytes).get diff --git a/chill-hadoop/src/main/java/com/twitter/chill/hadoop/KryoSerialization.java b/chill-hadoop/src/main/java/com/twitter/chill/hadoop/KryoSerialization.java index 29d1dc5a..e4826530 100644 --- a/chill-hadoop/src/main/java/com/twitter/chill/hadoop/KryoSerialization.java +++ b/chill-hadoop/src/main/java/com/twitter/chill/hadoop/KryoSerialization.java @@ -18,13 +18,13 @@ import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Output; +import org.objenesis.strategy.StdInstantiatorStrategy; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.serializer.Deserializer; import org.apache.hadoop.io.serializer.Serialization; import org.apache.hadoop.io.serializer.Serializer; -import org.objenesis.strategy.StdInstantiatorStrategy; import com.twitter.chill.KryoPool; import com.twitter.chill.KryoInstantiator; diff --git a/chill-hadoop/src/test/scala/com/twitter/chill/hadoop/HadoopTests.scala b/chill-hadoop/src/test/scala/com/twitter/chill/hadoop/HadoopTests.scala index 97f9cde8..a39d291f 100644 --- a/chill-hadoop/src/test/scala/com/twitter/chill/hadoop/HadoopTests.scala +++ b/chill-hadoop/src/test/scala/com/twitter/chill/hadoop/HadoopTests.scala @@ -17,8 +17,7 @@ limitations under the License. package com.twitter.chill.hadoop import com.esotericsoftware.kryo.Kryo - -import org.objenesis.strategy.StdInstantiatorStrategy +import org.objenesis.strategy.StdInstantiatorStrategy; import java.io.{ByteArrayInputStream => BAIn, ByteArrayOutputStream => BAOut} import org.apache.hadoop.conf.Configuration @@ -28,9 +27,18 @@ import com.twitter.chill.KryoInstantiator import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +class AnyKryoInstantiator extends KryoInstantiator { + override def newKryo: Kryo = { + val k = new Kryo + k.register(Class.forName("com.twitter.chill.hadoop.HadoopTests")) + k.register(Class.forName("scala.collection.immutable.List")) + k + } +} class StdKryoInstantiator extends KryoInstantiator { override def newKryo: Kryo = { val k = new Kryo + k.register(Class.forName("scala.Tuple2$mcII$sp")) k.setInstantiatorStrategy(new StdInstantiatorStrategy) k } @@ -56,7 +64,7 @@ class HadoopTests extends AnyWordSpec with Matchers { "accept anything" in { val conf = new Configuration val hc = new HadoopConfig(conf) - ConfiguredInstantiator.setReflect(hc, classOf[KryoInstantiator]) + ConfiguredInstantiator.setReflect(hc, classOf[AnyKryoInstantiator]) val ks = new KryoSerialization(conf) Seq(classOf[List[_]], classOf[Int], this.getClass).forall { cls => diff --git a/chill-java/src/main/java/com/twitter/chill/SerDeState.java b/chill-java/src/main/java/com/twitter/chill/SerDeState.java index 24cd403c..c5d9cbc8 100644 --- a/chill-java/src/main/java/com/twitter/chill/SerDeState.java +++ b/chill-java/src/main/java/com/twitter/chill/SerDeState.java @@ -43,7 +43,7 @@ protected SerDeState(Kryo k, Input in, Output out) { /** Call this when to reset the state to the initial state */ public void clear() { input.setBuffer(EMPTY_BUFFER); - output.clear(); + output.reset(); } public void setInput(byte[] in) { input.setBuffer(in); } diff --git a/chill-java/src/main/java/com/twitter/chill/java/ArraysAsListSerializer.java b/chill-java/src/main/java/com/twitter/chill/java/ArraysAsListSerializer.java index f4811d95..0d1ca9d9 100644 --- a/chill-java/src/main/java/com/twitter/chill/java/ArraysAsListSerializer.java +++ b/chill-java/src/main/java/com/twitter/chill/java/ArraysAsListSerializer.java @@ -71,7 +71,7 @@ public ArraysAsListSerializer() { } @Override - public List read(final Kryo kryo, final Input input, final Class> type) { + public List read(final Kryo kryo, final Input input, final Class> type) { final int length = input.readInt(true); Class componentType = kryo.readClass(input).getType(); try { diff --git a/chill-java/src/main/java/com/twitter/chill/java/BitSetSerializer.java b/chill-java/src/main/java/com/twitter/chill/java/BitSetSerializer.java index 597699c6..01994096 100644 --- a/chill-java/src/main/java/com/twitter/chill/java/BitSetSerializer.java +++ b/chill-java/src/main/java/com/twitter/chill/java/BitSetSerializer.java @@ -81,7 +81,7 @@ public void write(Kryo kryo, Output output, BitSet bitSet) { } @Override - public BitSet read(Kryo kryo, Input input, Class bitSetClass) { + public BitSet read(Kryo kryo, Input input, Class bitSetClass) { int len = input.readInt(true); long[] target = new long[len]; diff --git a/chill-java/src/main/java/com/twitter/chill/java/InetSocketAddressSerializer.java b/chill-java/src/main/java/com/twitter/chill/java/InetSocketAddressSerializer.java index c1515853..db11cf31 100644 --- a/chill-java/src/main/java/com/twitter/chill/java/InetSocketAddressSerializer.java +++ b/chill-java/src/main/java/com/twitter/chill/java/InetSocketAddressSerializer.java @@ -39,7 +39,7 @@ public void write(Kryo kryo, Output output, InetSocketAddress obj) { } @Override - public InetSocketAddress read(Kryo kryo, Input input, Class klass) { + public InetSocketAddress read(Kryo kryo, Input input, Class klass) { String host = input.readString(); int port = input.readInt(true); return new InetSocketAddress(host, port); diff --git a/chill-java/src/main/java/com/twitter/chill/java/IterableRegistrarSerializer.java b/chill-java/src/main/java/com/twitter/chill/java/IterableRegistrarSerializer.java index de61fe8b..153a3403 100644 --- a/chill-java/src/main/java/com/twitter/chill/java/IterableRegistrarSerializer.java +++ b/chill-java/src/main/java/com/twitter/chill/java/IterableRegistrarSerializer.java @@ -32,7 +32,7 @@ public void write(Kryo kryo, Output output, IterableRegistrar obj) { } kryo.writeClassAndObject(output, null); } - public IterableRegistrar read(Kryo kryo, Input input, Class type) { + public IterableRegistrar read(Kryo kryo, Input input, Class type) { ArrayList krs = new ArrayList(); IKryoRegistrar thisKr = (IKryoRegistrar)kryo.readClassAndObject(input); while(thisKr != null) { diff --git a/chill-java/src/main/java/com/twitter/chill/java/PriorityQueueSerializer.java b/chill-java/src/main/java/com/twitter/chill/java/PriorityQueueSerializer.java index c9dd95fd..a9af4253 100644 --- a/chill-java/src/main/java/com/twitter/chill/java/PriorityQueueSerializer.java +++ b/chill-java/src/main/java/com/twitter/chill/java/PriorityQueueSerializer.java @@ -62,7 +62,7 @@ public void write(Kryo k, Output o, PriorityQueue q) { o.flush(); } } - public PriorityQueue read(Kryo k, Input i, Class> c) { + public PriorityQueue read(Kryo k, Input i, Class> c) { Comparator comp = (Comparator)k.readClassAndObject(i); int sz = i.readInt(true); // can't create with size 0: diff --git a/chill-java/src/main/java/com/twitter/chill/java/RegexSerializer.java b/chill-java/src/main/java/com/twitter/chill/java/RegexSerializer.java index 2f9742ab..c23c41fe 100644 --- a/chill-java/src/main/java/com/twitter/chill/java/RegexSerializer.java +++ b/chill-java/src/main/java/com/twitter/chill/java/RegexSerializer.java @@ -38,7 +38,7 @@ public void write(Kryo kryo, Output output, Pattern pattern) { } @Override - public Pattern read(Kryo kryo, Input input, Class patternClass) { + public Pattern read(Kryo kryo, Input input, Class patternClass) { return Pattern.compile(input.readString()); } } diff --git a/chill-java/src/main/java/com/twitter/chill/java/SqlDateSerializer.java b/chill-java/src/main/java/com/twitter/chill/java/SqlDateSerializer.java index d31061df..e9814b19 100644 --- a/chill-java/src/main/java/com/twitter/chill/java/SqlDateSerializer.java +++ b/chill-java/src/main/java/com/twitter/chill/java/SqlDateSerializer.java @@ -38,7 +38,7 @@ public void write(Kryo kryo, Output output, Date date) { } @Override - public Date read(Kryo kryo, Input input, Class dateClass) { + public Date read(Kryo kryo, Input input, Class dateClass) { return new Date(input.readLong(true)); } } diff --git a/chill-java/src/main/java/com/twitter/chill/java/SqlTimeSerializer.java b/chill-java/src/main/java/com/twitter/chill/java/SqlTimeSerializer.java index eeb5e938..37729bcf 100644 --- a/chill-java/src/main/java/com/twitter/chill/java/SqlTimeSerializer.java +++ b/chill-java/src/main/java/com/twitter/chill/java/SqlTimeSerializer.java @@ -38,7 +38,7 @@ public void write(Kryo kryo, Output output, Time time) { } @Override - public Time read(Kryo kryo, Input input, Class