Merge pull request #2676 from square/py/improve_heap_diff

Iterating on heap growth based on friction / feedback from trying to integrate it inside Square.

gradle/libs.versions.toml

@@ -41,6 +41,7 @@ kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref =
 # and they'll automatically resolve to higher version without having to necessarily resort to a
 # resolution strategy.
 androidX-fragment = { module = "androidx.fragment:fragment", version = "1.0.0" }
+androidX-multidex = { module = "androidx.multidex:multidex", version = "2.0.1" }
 # Exposed transitively, avoid increasing
 androidX-startup = { module = "androidx.startup:startup-runtime", version = "1.0.0" }
 androidX-test-core = { module = "androidx.test:core", version = "1.4.0" }
@@ -55,6 +56,7 @@ androidX-test-junitKtx = { module = "androidx.test.ext:junit-ktx", version.ref =
 androidX-test-uiautomator = { module = "androidx.test.uiautomator:uiautomator", version = "2.2.0" }
 androidX-work-runtime = { module = "androidx.work:work-runtime", version.ref = "workManager" }
 androidX-work-multiprocess = { module = "androidx.work:work-multiprocess", version.ref = "workManager" }
+androidX-collections = { module = "androidx.collection:collection-ktx", version = "1.4.0" }
 
 androidSupport = { module = "com.android.support:support-v4", version = "28.0.0" }
 assertjCore = { module = "org.assertj:assertj-core", version = "3.9.1" }

leakcanary/leakcanary-android-instrumentation/build.gradle

@@ -15,6 +15,7 @@ dependencies {
   androidTestImplementation projects.objectWatcher.objectWatcherAndroid
   // Plumber auto installer for running tests
   androidTestImplementation projects.plumber.plumberAndroid
+  androidTestImplementation libs.androidX.multidex
   androidTestImplementation libs.androidX.test.core
   androidTestImplementation libs.androidX.test.espresso
   androidTestImplementation libs.androidX.test.rules
@@ -28,6 +29,7 @@ android {
     targetSdk versions.compileSdk
     minSdk versions.minSdk
     testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    multiDexEnabled true
   }
   buildFeatures.buildConfig = false
   namespace 'com.squareup.leakcanary.instrumentation'

leakcanary/leakcanary-android-instrumentation/src/androidTest/AndroidManifest.xml

@@ -16,7 +16,9 @@
   -->
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">
 
-  <application>
+  <application
+    android:name="androidx.multidex.MultiDexApplication"
+    >
     <activity android:name="leakcanary.TestActivity"/>
   </application>
 </manifest>

leakcanary/leakcanary-android-test/src/main/java/leakcanary/RepeatingAndroidInProcessScenario.kt

@@ -1,8 +1,8 @@
 package leakcanary
 
-import shark.RepeatingScenarioObjectGrowthDetector
 import shark.HeapGraphProvider
 import shark.ObjectGrowthDetector
+import shark.RepeatingScenarioObjectGrowthDetector
 import shark.repeatingScenario
 
 /**
@@ -23,7 +23,8 @@ fun ObjectGrowthDetector.repeatingAndroidInProcessScenario(
   return repeatingScenario(
     heapGraphProvider = HeapGraphProvider.dumpingAndDeleting(
       heapDumper = HeapDumper.forAndroidInProcess()
-        .withGc(gcTrigger = GcTrigger.inProcess()),
+        .withGc(gcTrigger = GcTrigger.inProcess())
+        .withDetectorWarmup(this),
       heapDumpFileProvider = HeapDumpFileProvider.tempFile()
     ),
     maxHeapDumps = maxHeapDumps,

leakcanary/leakcanary-core/api/leakcanary-core.api

@@ -49,6 +49,19 @@ public final class leakcanary/HeapDumperKt {
 	public static synthetic fun withGc$default (Lleakcanary/HeapDumper;Lleakcanary/GcTrigger;ILjava/lang/Object;)Lleakcanary/HeapDumper;
 }
 
+public final class leakcanary/ObjectGrowthWarmupHeapDumper : leakcanary/HeapDumper {
+	public static final field Companion Lleakcanary/ObjectGrowthWarmupHeapDumper$Companion;
+	public fun <init> (Lshark/ObjectGrowthDetector;Lleakcanary/HeapDumper;)V
+	public fun dumpHeap (Ljava/io/File;)V
+}
+
+public final class leakcanary/ObjectGrowthWarmupHeapDumper$Companion {
+}
+
+public final class leakcanary/ObjectGrowthWarmupHeapDumperKt {
+	public static final fun withDetectorWarmup (Lleakcanary/HeapDumper;Lshark/ObjectGrowthDetector;)Lleakcanary/HeapDumper;
+}
+
 public final class leakcanary/TempHeapDumpFileProvider : leakcanary/HeapDumpFileProvider {
 	public static final field INSTANCE Lleakcanary/TempHeapDumpFileProvider;
 	public fun newHeapDumpFile ()Ljava/io/File;

leakcanary/leakcanary-core/build.gradle

@@ -9,4 +9,9 @@ targetCompatibility = JavaVersion.VERSION_1_8
 dependencies {
   api projects.leakcanary.leakcanaryGc
   api projects.shark.shark
+  implementation libs.okio2
+
+  testImplementation libs.assertjCore
+  testImplementation libs.junit
+  testImplementation projects.shark.sharkHprofTest
 }

leakcanary/leakcanary-core/src/main/java/leakcanary/ObjectGrowthWarmupHeapDumper.kt

@@ -0,0 +1,59 @@
+package leakcanary
+
+import java.io.File
+import shark.ByteArraySourceProvider
+import shark.ObjectGrowthDetector
+import shark.repeatingScenario
+import okio.ByteString.Companion.decodeHex
+import shark.HprofHeapGraph.Companion.openHeapGraph
+
+class ObjectGrowthWarmupHeapDumper(
+  private val objectGrowthDetector: ObjectGrowthDetector,
+  private val delegate: HeapDumper
+) : HeapDumper {
+
+  private var warm = false
+
+  override fun dumpHeap(heapDumpFile: File) {
+    if (!warm) {
+      warmup()
+      warm = true
+    }
+    delegate.dumpHeap(heapDumpFile)
+  }
+
+  private fun warmup() {
+    val heapDumpsAsHex = listOf({ heapDump1Hex() }, { heapDump2Hex() }, { heapDump3Hex() })
+    val heapDumpsAsHexIterator = heapDumpsAsHex.iterator()
+    val warmupDetector = objectGrowthDetector.repeatingScenario(
+      heapGraphProvider = {
+        ByteArraySourceProvider(
+          heapDumpsAsHexIterator.next()().decodeHex().toByteArray()
+        ).openHeapGraph()
+      },
+      maxHeapDumps = heapDumpsAsHex.size,
+      scenarioLoopsPerDump = 1
+    )
+    warmupDetector.findRepeatedlyGrowingObjects {}
+  }
+
+  @SuppressWarnings("MaxLineLength")
+  companion object {
+    // Header:
+    // 4a4156412050524f46494c4520312e302e33 is the header version string
+    // then 00 is the string separator
+    // 00000004 is the identifier byte size, 4 bytes
+    // 0b501e7e ca55e77e (obsolete cassette) is a cool heap dump timestamp.
+    internal fun heapDump1Hex() =
+      "4a4156412050524f46494c4520312e302e3300000000040b501e7eca55e77e01000000000000001b000000016a6176612e6c616e672e7265662e5265666572656e63650200000000000000100000000100000002000000010000000101000000000000000c000000037265666572656e74010000000000000014000000046a6176612e6c616e672e4f626a656374020000000000000010000000010000000500000001000000040c000000000000006520000000050000000100000000000000000000000000000000000000000000000000000000000000000000050000000520000000020000000100000005000000000000000000000000000000000000000000000004000000000001000000030205000000022c000000000000000001000000000000001f000000066a6176612e6c616e672e7265662e5765616b5265666572656e6365020000000000000010000000010000000700000001000000060c00000000000000302000000007000000010000000200000000000000000000000000000000000000000000000400000000000005000000072c0000000000000000010000000000000021000000086c65616b63616e6172792e4b657965645765616b5265666572656e6365020000000000000010000000010000000900000001000000080100000000000000180000000a6865617044756d70557074696d654d696c6c69730100000000000000070000000b6b65790100000000000000080000000c6e616d650100000000000000150000000d7761746368557074696d654d696c6c69730100000000000000180000000e72657461696e6564557074696d654d696c6c69730c00000000000000622000000009000000010000000700000000000000000000000000000000000000000000001c000000010000000a0b000000000000753000040000000b020000000c020000000d0b0000000e0b0500000009210000000f0000000100000005000000002c0000000000000000010000000000000016000000106a6176612e6c616e672e4f626a6563745b5d020000000000000010000000010000001100000001000000100c000000000000004520000000110000000100000005000000000000000000000000000000000000000000000000000000000000050000001122000000120000000100000001000000110000000f2c000000000000000001000000000000000a00000013486f6c64657202000000000000001000000001000000140000000100000013010000000000000008000000156c6973740c00000000000000392000000014000000010000000500000000000000000000000000000000000000000000000000000001000000150200000012000005000000142c0000000000000000"
+
+    internal fun heapDump2Hex() =
+      "4a4156412050524f46494c4520312e302e3300000000040b501e7eca55e77e01000000000000001b000000016a6176612e6c616e672e7265662e5265666572656e63650200000000000000100000000100000002000000010000000101000000000000000c000000037265666572656e74010000000000000014000000046a6176612e6c616e672e4f626a656374020000000000000010000000010000000500000001000000040c000000000000006520000000050000000100000000000000000000000000000000000000000000000000000000000000000000050000000520000000020000000100000005000000000000000000000000000000000000000000000004000000000001000000030205000000022c000000000000000001000000000000001f000000066a6176612e6c616e672e7265662e5765616b5265666572656e6365020000000000000010000000010000000700000001000000060c00000000000000302000000007000000010000000200000000000000000000000000000000000000000000000400000000000005000000072c0000000000000000010000000000000021000000086c65616b63616e6172792e4b657965645765616b5265666572656e6365020000000000000010000000010000000900000001000000080100000000000000180000000a6865617044756d70557074696d654d696c6c69730100000000000000070000000b6b65790100000000000000080000000c6e616d650100000000000000150000000d7761746368557074696d654d696c6c69730100000000000000180000000e72657461696e6564557074696d654d696c6c69730c00000000000000732000000009000000010000000700000000000000000000000000000000000000000000001c000000010000000a0b000000000000753000040000000b020000000c020000000d0b0000000e0b0500000009210000000f00000001000000050000000021000000100000000100000005000000002c0000000000000000010000000000000016000000116a6176612e6c616e672e4f626a6563745b5d020000000000000010000000010000001200000001000000110c000000000000004920000000120000000100000005000000000000000000000000000000000000000000000000000000000000050000001222000000130000000100000002000000120000000f000000102c000000000000000001000000000000000a00000014486f6c64657202000000000000001000000001000000150000000100000014010000000000000008000000166c6973740c00000000000000392000000015000000010000000500000000000000000000000000000000000000000000000000000001000000160200000013000005000000152c0000000000000000"
+
+    internal fun heapDump3Hex() =
+      "4a4156412050524f46494c4520312e302e3300000000040b501e7eca55e77e01000000000000001b000000016a6176612e6c616e672e7265662e5265666572656e63650200000000000000100000000100000002000000010000000101000000000000000c000000037265666572656e74010000000000000014000000046a6176612e6c616e672e4f626a656374020000000000000010000000010000000500000001000000040c000000000000006520000000050000000100000000000000000000000000000000000000000000000000000000000000000000050000000520000000020000000100000005000000000000000000000000000000000000000000000004000000000001000000030205000000022c000000000000000001000000000000001f000000066a6176612e6c616e672e7265662e5765616b5265666572656e6365020000000000000010000000010000000700000001000000060c00000000000000302000000007000000010000000200000000000000000000000000000000000000000000000400000000000005000000072c0000000000000000010000000000000021000000086c65616b63616e6172792e4b657965645765616b5265666572656e6365020000000000000010000000010000000900000001000000080100000000000000180000000a6865617044756d70557074696d654d696c6c69730100000000000000070000000b6b65790100000000000000080000000c6e616d650100000000000000150000000d7761746368557074696d654d696c6c69730100000000000000180000000e72657461696e6564557074696d654d696c6c69730c00000000000000842000000009000000010000000700000000000000000000000000000000000000000000001c000000010000000a0b000000000000753000040000000b020000000c020000000d0b0000000e0b0500000009210000000f000000010000000500000000210000001000000001000000050000000021000000110000000100000005000000002c0000000000000000010000000000000016000000126a6176612e6c616e672e4f626a6563745b5d020000000000000010000000010000001300000001000000120c000000000000004d20000000130000000100000005000000000000000000000000000000000000000000000000000000000000050000001322000000140000000100000003000000130000000f00000010000000112c000000000000000001000000000000000a00000015486f6c64657202000000000000001000000001000000160000000100000015010000000000000008000000176c6973740c00000000000000392000000016000000010000000500000000000000000000000000000000000000000000000000000001000000170200000014000005000000162c0000000000000000"
+  }
+}
+
+fun HeapDumper.withDetectorWarmup(objectGrowthDetector: ObjectGrowthDetector): HeapDumper =
+  ObjectGrowthWarmupHeapDumper(objectGrowthDetector, this)

leakcanary/leakcanary-core/src/test/java/leakcanary/ObjectGrowthWarmupHeapDumperTest.kt

@@ -0,0 +1,48 @@
+package leakcanary
+
+import okio.ByteString.Companion.decodeHex
+import okio.ByteString.Companion.toByteString
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.Test
+import shark.HprofHeader
+import shark.dumpToBytes
+
+class ObjectGrowthWarmupHeapDumperTest {
+
+  @Test fun `heap dump 1 as hex constant matches generated heap dump hex`() {
+    assertThat(ObjectGrowthWarmupHeapDumper.heapDump1Hex()).isEqualTo(dumpGrowingListHeapAsHex(1))
+  }
+
+  @Test fun `heap dump 2 as hex constant matches generated heap dump hex`() {
+    assertThat(ObjectGrowthWarmupHeapDumper.heapDump2Hex()).isEqualTo(dumpGrowingListHeapAsHex(2))
+  }
+
+  @Test fun `heap dump 3 as hex constant matches generated heap dump hex`() {
+    assertThat(ObjectGrowthWarmupHeapDumper.heapDump3Hex()).isEqualTo(dumpGrowingListHeapAsHex(3))
+  }
+
+  private fun dumpGrowingListHeapAsHex(listItemCount: Int): String {
+    val heapDumpTimestamp = ("0b501e7e" + "ca55e77e").decodeHex().toByteArray().toLong()
+    return dumpToBytes(hprofHeader = HprofHeader(heapDumpTimestamp = heapDumpTimestamp)) {
+      "Holder" clazz {
+        val refs = (1..listItemCount).map {
+          instance(objectClassId)
+        }.toTypedArray()
+        staticField["list"] = objectArray(*refs)
+      }
+    }.toByteString().hex()
+  }
+
+  private fun ByteArray.toLong(): Long {
+    check(size == 8)
+    var pos = 0
+    return (this[pos++].toLong() and 0xffL shl 56
+      or (this[pos++].toLong() and 0xffL shl 48)
+      or (this[pos++].toLong() and 0xffL shl 40)
+      or (this[pos++].toLong() and 0xffL shl 32)
+      or (this[pos++].toLong() and 0xffL shl 24)
+      or (this[pos++].toLong() and 0xffL shl 16)
+      or (this[pos++].toLong() and 0xffL shl 8)
+      or (this[pos].toLong() and 0xffL))
+  }
+}

leakcanary/leakcanary-jvm-test/build.gradle

@@ -12,4 +12,5 @@ dependencies {
 
   testImplementation libs.assertjCore
   testImplementation libs.junit
+  testImplementation projects.shark.sharkHprofTest
 }

leakcanary/leakcanary-jvm-test/src/main/java/leakcanary/RepeatingJvmInProcessScenario.kt

@@ -23,7 +23,8 @@ fun ObjectGrowthDetector.repeatingJvmInProcessScenario(
   return repeatingScenario(
     heapGraphProvider = HeapGraphProvider.dumpingAndDeleting(
       heapDumper = HeapDumper.forJvmInProcess()
-        .withGc(gcTrigger = GcTrigger.inProcess()),
+        .withGc(gcTrigger = GcTrigger.inProcess())
+        .withDetectorWarmup(this),
       heapDumpFileProvider = HeapDumpFileProvider.tempFile()
     ),
     maxHeapDumps = maxHeapDumps,

leakcanary/leakcanary-jvm-test/src/test/java/leakcanary/JvmHeapGrowthDetectorConfigTest.kt

@@ -2,9 +2,7 @@ package leakcanary
 
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.Test
-import shark.HeapGraphProvider
 import shark.ObjectGrowthDetector
-import shark.repeatingScenario
 import shark.forJvmHeap
 
 class JvmHeapGrowthDetectorConfigTest {
@@ -15,7 +13,10 @@ class JvmHeapGrowthDetectorConfigTest {
 
   @Test
   fun `leaky increase leads to heap growth`() {
-    val detector = ObjectGrowthDetector.forJvmHeap().repeatingJvmInProcessScenario()
+    val detector = ObjectGrowthDetector.forJvmHeap().repeatingJvmInProcessScenario(
+      maxHeapDumps = 2,
+      scenarioLoopsPerDump = 10
+    )
 
     val growingNodes = detector.findRepeatedlyGrowingObjects {
       leakies += Leaky()