From 889005ac592c8421df8be009afc922d7ccf691ce Mon Sep 17 00:00:00 2001 From: Jian Guo Date: Wed, 21 Jun 2023 17:12:00 +0800 Subject: [PATCH] hprof parser and inspector (#325) * feat: hprof parser * fix: fix build: * fix: fix build break * feat: check dumpheap return value * feat: check only if app is debuggable * fix: fix merge issues * feat: only dump hprof with debugable build * fix: fix build * feat: shorten hprof file name * feat: remove appid in file name * feat: add final * feat: remove unused code --- common/build.gradle | 2 + .../PerformanceTestManagementService.java | 15 +- .../entity/AndroidHprofMemoryInfo.java | 19 + .../performance/entity/AndroidMemoryInfo.java | 2 + .../performance/hprof/BitmapInfo.java | 27 ++ .../hprof/BitmapInfoExtractor.java | 19 + .../hydralab/performance/hprof/Extractor.java | 176 +++++++++ .../performance/hprof/HeapProfProcessor.java | 350 ++++++++++++++++++ .../performance/hprof/ObjectInfo.java | 67 ++++ .../hprof/TopObjectInfoExtractor.java | 41 ++ .../AndroidMemoryHprofInspector.java | 70 ++++ .../AndroidMemoryInfoInspector.java | 3 + .../AndroidMemoryHprofResultParser.java | 68 ++++ .../AndroidMemoryInfoResultParser.java | 3 + sdk/build.gradle | 1 + .../PerformanceInspectionResult.java | 3 + .../performance/PerformanceResultParser.java | 5 +- 17 files changed, 867 insertions(+), 4 deletions(-) create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/entity/AndroidHprofMemoryInfo.java create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/hprof/BitmapInfo.java create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/hprof/BitmapInfoExtractor.java create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/hprof/Extractor.java create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/hprof/HeapProfProcessor.java create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/hprof/ObjectInfo.java create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/hprof/TopObjectInfoExtractor.java create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidMemoryHprofInspector.java create mode 100644 common/src/main/java/com/microsoft/hydralab/performance/parsers/AndroidMemoryHprofResultParser.java diff --git a/common/build.gradle b/common/build.gradle index 1696b01d8..28bd8f52b 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -72,6 +72,8 @@ dependencies { compile group: 'net.dongliu', name: 'apk-parser', version: '2.5.3' //Ipa Parse compile group: 'com.googlecode.plist', name: 'dd-plist', version: '1.3' + + compile 'com.squareup.haha:haha:2.1' } repositories { diff --git a/common/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java b/common/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java index eb9218f10..44d45936b 100644 --- a/common/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java +++ b/common/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java @@ -13,12 +13,16 @@ import com.microsoft.hydralab.common.util.FileUtil; import com.microsoft.hydralab.common.util.ThreadPoolUtil; import com.microsoft.hydralab.performance.inspectors.AndroidBatteryInfoInspector; +import com.microsoft.hydralab.performance.inspectors.AndroidMemoryHprofInspector; +import com.microsoft.hydralab.performance.inspectors.AndroidMemoryHprofInspector; import com.microsoft.hydralab.performance.inspectors.AndroidMemoryInfoInspector; import com.microsoft.hydralab.performance.inspectors.IOSEnergyGaugeInspector; import com.microsoft.hydralab.performance.inspectors.IOSMemoryPerfInspector; import com.microsoft.hydralab.performance.inspectors.WindowsBatteryInspector; import com.microsoft.hydralab.performance.inspectors.WindowsMemoryInspector; import com.microsoft.hydralab.performance.parsers.AndroidBatteryInfoResultParser; +import com.microsoft.hydralab.performance.parsers.AndroidMemoryHprofResultParser; +import com.microsoft.hydralab.performance.parsers.AndroidMemoryHprofResultParser; import com.microsoft.hydralab.performance.parsers.AndroidMemoryInfoResultParser; import com.microsoft.hydralab.performance.parsers.IOSEnergyGaugeResultParser; import com.microsoft.hydralab.performance.parsers.IOSMemoryPerfResultParser; @@ -39,6 +43,7 @@ import java.util.concurrent.ScheduledFuture; import static com.microsoft.hydralab.performance.PerformanceInspector.PerformanceInspectorType.INSPECTOR_ANDROID_BATTERY_INFO; +import static com.microsoft.hydralab.performance.PerformanceInspector.PerformanceInspectorType.INSPECTOR_ANDROID_MEMORY_DUMP; import static com.microsoft.hydralab.performance.PerformanceInspector.PerformanceInspectorType.INSPECTOR_ANDROID_MEMORY_INFO; import static com.microsoft.hydralab.performance.PerformanceInspector.PerformanceInspectorType.INSPECTOR_EVENT_TIME; import static com.microsoft.hydralab.performance.PerformanceInspector.PerformanceInspectorType.INSPECTOR_IOS_ENERGY; @@ -46,6 +51,7 @@ import static com.microsoft.hydralab.performance.PerformanceInspector.PerformanceInspectorType.INSPECTOR_WIN_BATTERY; import static com.microsoft.hydralab.performance.PerformanceInspector.PerformanceInspectorType.INSPECTOR_WIN_MEMORY; import static com.microsoft.hydralab.performance.PerformanceResultParser.PerformanceResultParserType.PARSER_ANDROID_BATTERY_INFO; +import static com.microsoft.hydralab.performance.PerformanceResultParser.PerformanceResultParserType.PARSER_ANDROID_MEMORY_DUMP; import static com.microsoft.hydralab.performance.PerformanceResultParser.PerformanceResultParserType.PARSER_ANDROID_MEMORY_INFO; import static com.microsoft.hydralab.performance.PerformanceResultParser.PerformanceResultParserType.PARSER_EVENT_TIME; import static com.microsoft.hydralab.performance.PerformanceResultParser.PerformanceResultParserType.PARSER_IOS_ENERGY; @@ -57,18 +63,20 @@ public class PerformanceTestManagementService implements IPerformanceInspectionS private static final String PERFORMANCE_FOLDER_NAME = "performance"; private static final Map inspectorParserTypeMap = Map.of( INSPECTOR_ANDROID_BATTERY_INFO, PARSER_ANDROID_BATTERY_INFO, - INSPECTOR_ANDROID_MEMORY_INFO, PARSER_ANDROID_MEMORY_INFO, INSPECTOR_WIN_MEMORY, PARSER_WIN_MEMORY, INSPECTOR_WIN_BATTERY, PARSER_WIN_BATTERY, INSPECTOR_IOS_ENERGY, PARSER_IOS_ENERGY, INSPECTOR_IOS_MEMORY, PARSER_IOS_MEMORY, + INSPECTOR_ANDROID_MEMORY_INFO, PARSER_ANDROID_MEMORY_INFO, + INSPECTOR_ANDROID_MEMORY_DUMP, PARSER_ANDROID_MEMORY_DUMP, INSPECTOR_EVENT_TIME, PARSER_EVENT_TIME ); private final Map performanceInspectorMap = Map.of( INSPECTOR_ANDROID_BATTERY_INFO, new AndroidBatteryInfoInspector(), - INSPECTOR_ANDROID_MEMORY_INFO, new AndroidMemoryInfoInspector(), INSPECTOR_WIN_MEMORY, new WindowsMemoryInspector(), INSPECTOR_WIN_BATTERY, new WindowsBatteryInspector(), + INSPECTOR_ANDROID_MEMORY_INFO, new AndroidMemoryInfoInspector(), + INSPECTOR_ANDROID_MEMORY_DUMP, new AndroidMemoryHprofInspector(), INSPECTOR_IOS_MEMORY, new IOSMemoryPerfInspector(), INSPECTOR_IOS_ENERGY, new IOSEnergyGaugeInspector() ); @@ -81,10 +89,11 @@ INSPECTOR_IOS_ENERGY, new IOSEnergyGaugeInspector() INSPECTOR_IOS_ENERGY, DeviceType.IOS ); private final Map performanceResultParserMap = Map.of( - PARSER_ANDROID_MEMORY_INFO, new AndroidMemoryInfoResultParser(), PARSER_ANDROID_BATTERY_INFO, new AndroidBatteryInfoResultParser(), PARSER_WIN_MEMORY, new WindowsMemoryResultParser(), PARSER_WIN_BATTERY, new WindowsBatteryResultParser(), + PARSER_ANDROID_MEMORY_INFO, new AndroidMemoryInfoResultParser(), + PARSER_ANDROID_MEMORY_DUMP, new AndroidMemoryHprofResultParser(), PARSER_IOS_ENERGY, new IOSEnergyGaugeResultParser(), PARSER_IOS_MEMORY, new IOSMemoryPerfResultParser() ); diff --git a/common/src/main/java/com/microsoft/hydralab/performance/entity/AndroidHprofMemoryInfo.java b/common/src/main/java/com/microsoft/hydralab/performance/entity/AndroidHprofMemoryInfo.java new file mode 100644 index 000000000..cfca30f0a --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/entity/AndroidHprofMemoryInfo.java @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.entity; + +import com.microsoft.hydralab.performance.hprof.ObjectInfo; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +@Data +public class AndroidHprofMemoryInfo implements Serializable { + private List bitmapInfoList; + private List topObjectList; + private String appPackageName; + private long timeStamp; + private String description; +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/entity/AndroidMemoryInfo.java b/common/src/main/java/com/microsoft/hydralab/performance/entity/AndroidMemoryInfo.java index b1676d571..22c0ac091 100644 --- a/common/src/main/java/com/microsoft/hydralab/performance/entity/AndroidMemoryInfo.java +++ b/common/src/main/java/com/microsoft/hydralab/performance/entity/AndroidMemoryInfo.java @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. package com.microsoft.hydralab.performance.entity; import com.alibaba.fastjson.annotation.JSONField; diff --git a/common/src/main/java/com/microsoft/hydralab/performance/hprof/BitmapInfo.java b/common/src/main/java/com/microsoft/hydralab/performance/hprof/BitmapInfo.java new file mode 100644 index 000000000..df3cb83cd --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/hprof/BitmapInfo.java @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.hprof; + +import java.io.Serializable; + +public class BitmapInfo extends ObjectInfo implements Serializable { + + public int width; + public int height; + public int density; + public boolean recycled; + public int pixelsCount; + public long nativePtr; + public float perPixelSize; + + + public void computePerPixelSize() { + perPixelSize = nativeSize * 1f / height / width; + } + + @Override + public String getSizeInfo() { + return super.getSizeInfo() + ",  BitmapSize: " + width + " × " + height; + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/hprof/BitmapInfoExtractor.java b/common/src/main/java/com/microsoft/hydralab/performance/hprof/BitmapInfoExtractor.java new file mode 100644 index 000000000..9cd6a2e2d --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/hprof/BitmapInfoExtractor.java @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.hprof; + +import com.squareup.haha.perflib.Instance; + +public class BitmapInfoExtractor extends Extractor { + + @Override + public ObjectInfo extractInstanceInfo(int retainedSizeRanking, Instance instance) { + return extractBitmapInfo(retainedSizeRanking, instance); + } + + @Override + public String getType() { + return "bitmap"; + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/hprof/Extractor.java b/common/src/main/java/com/microsoft/hydralab/performance/hprof/Extractor.java new file mode 100644 index 000000000..c9570de39 --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/hprof/Extractor.java @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.hprof; + +import com.squareup.haha.perflib.ClassInstance; +import com.squareup.haha.perflib.ClassObj; +import com.squareup.haha.perflib.Field; +import com.squareup.haha.perflib.Instance; +import com.squareup.haha.perflib.Type; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public abstract class Extractor { + /** + * min size in byte + */ + private int minNativeSize = 1000; + + protected List resultList = new ArrayList<>(); + + public void setMinNativeSize(int minNativeSize) { + this.minNativeSize = minNativeSize; + } + + public String getName() { + return this.getClass().getSimpleName(); + } + + void onExtractInfo(int retainedSizeRanking, Instance instance) { + ObjectInfo objectInfo = extractInstanceInfo(retainedSizeRanking, instance); + if (objectInfo == null) { + return; + } + resultList.add(objectInfo); + } + + abstract ObjectInfo extractInstanceInfo(int retainedSizeRanking, Instance instance); + + public List getResultList() { + return resultList; + } + + protected void mapSetBaseAttr(Instance instance, ObjectInfo bitmapInfo) { + List founds = new ArrayList<>(); + ClassObj classObj = instance.getClassObj(); + String className = classObj.getClassName(); + String[] names = new String[1]; + findRelatedInstances(instance, null, founds, names); + bitmapInfo.instance = instance; + bitmapInfo.firstLevelLauncherRef = founds.size() > 0 ? founds.get(0) : null; + bitmapInfo.className = className; + if (bitmapInfo.firstLevelLauncherRef instanceof ClassObj) { + bitmapInfo.isStaticMember = true; + } + bitmapInfo.fieldName = names[0]; + bitmapInfo.nativeSize = instance.getNativeSize(); + bitmapInfo.distanceToRoot = instance.getDistanceToGcRoot(); + bitmapInfo.retainedSize = instance.getTotalRetainedSize(); + bitmapInfo.size = instance.getSize(); + bitmapInfo.id = instance.getId(); + bitmapInfo.uniqueId = instance.getUniqueId(); + } + + protected void findRelatedInstances(Instance instance, Instance visited, List founds, String[] names) { + if (instance == null) { + return; + } + ClassObj classObj = instance.getClassObj(); + if (classObj == null) { + if (!(instance instanceof ClassObj)) { + return; + } + classObj = (ClassObj) instance; + } + String className = classObj.getClassName(); + if (className.contains(".launcher")) { + founds.add(instance); + if (instance instanceof ClassInstance) { + ClassInstance classInstance = (ClassInstance) instance; + List values = classInstance.getValues(); + for (ClassInstance.FieldValue value : values) { + if (Objects.equals(value.getField().getType(), Type.OBJECT)) { + if (Objects.equals(value.getValue(), visited)) { + if (visited != null) { + founds.add(visited); + names[0] = value.getField().getName(); + } + } + } + } + // static case + } else if (instance instanceof ClassObj) { + if (instance.getNextInstanceToGcRoot() == null) { + founds.add(instance); + Map staticFieldValues = ((ClassObj) instance).getStaticFieldValues(); + for (Map.Entry fieldObjectEntry : staticFieldValues.entrySet()) { + Field key = fieldObjectEntry.getKey(); + if (Objects.equals(key.getType(), Type.OBJECT)) { + if (Objects.equals(staticFieldValues.get(key), visited)) { + if (visited != null) { + founds.add(visited); + names[0] = key.getName(); + } + } + } + } + } + } + return; + } + findRelatedInstances(instance.getNextInstanceToGcRoot(), instance, founds, names); + } + + public ObjectInfo extractBitmapInfo(int retainedSizeRanking, Instance instance) { + ClassObj classObj = instance.getClassObj(); + String className = classObj.getClassName(); + if (className.equals("android.graphics.Bitmap")) { + if (instance instanceof ClassInstance) { + ClassInstance classInstance = (ClassInstance) instance; + List values = classInstance.getValues(); + BitmapInfo bitmapInfo = new BitmapInfo(); + if (instance.getNativeSize() < minNativeSize) { + return null; + } + mapSetBaseAttr(instance, bitmapInfo); + for (ClassInstance.FieldValue value : values) { + Field field = value.getField(); + String name = field.getName(); + switch (name) { + case "mWidth": + bitmapInfo.width = (int) value.getValue(); + break; + case "mHeight": + bitmapInfo.height = (int) value.getValue(); + break; + case "mDensity": + bitmapInfo.density = (int) value.getValue(); + break; + case "mNativePtr": + bitmapInfo.nativePtr = (long) value.getValue(); + break; + case "mRecycled": + bitmapInfo.recycled = (boolean) value.getValue(); + break; + } + } + bitmapInfo.computePerPixelSize(); + return bitmapInfo; + + } + } + return null; + } + + public abstract String getType(); + + public void onExtractComplete() { + resultList.sort((o1, o2) -> Long.compare(o2.retainedSize, o1.retainedSize)); + Iterator iterator = resultList.iterator(); + int i = 0; + while (iterator.hasNext()) { + ObjectInfo next = iterator.next(); + if (next.distanceToRoot == 0 || next.getFieldChainString() == null) { + iterator.remove(); + continue; + } + next.index = i + 1; + i++; + } + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/hprof/HeapProfProcessor.java b/common/src/main/java/com/microsoft/hydralab/performance/hprof/HeapProfProcessor.java new file mode 100644 index 000000000..6b8929e41 --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/hprof/HeapProfProcessor.java @@ -0,0 +1,350 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.hprof; + +import com.android.tools.perflib.captures.DataBuffer; +import com.android.tools.perflib.captures.MemoryMappedFileBuffer; +import com.squareup.haha.perflib.*; +import gnu.trove.THashMap; +import gnu.trove.TObjectProcedure; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static com.microsoft.hydralab.common.util.FileUtil.getSizeStringWithTagIfLarge; + +public class HeapProfProcessor { + public static final int MAX_FIELD_CHAIN_DEPTH = 10; + //The max length that channel can display + public static final int HTML_MAX_LENGTH = 18000; + + private final File heapDumpFile; + private static Logger logger = LoggerFactory.getLogger(HeapProfProcessor.class.getSimpleName()); + private HashMap extractorMap = new HashMap<>(); + + public HeapProfProcessor(File heapFile) { + heapDumpFile = heapFile; + } + + private static String generateRootKey(RootObj root) { + return String.format("%s@0x%08x", root.getRootType().getName(), root.getId()); + } + + /** + * TODO + * + * @param instance + * @return + */ + public static String getObjectRefTrace(Instance instance) { + return "todo"; + } + + public void registerExtractor(Extractor extractor) { + extractorMap.put(extractor.getName(), extractor); + } + + + public void loadAndExtract() throws IOException { + if (!heapDumpFile.exists()) { + return; + } + + DataBuffer buffer = new MemoryMappedFileBuffer(heapDumpFile); + + Snapshot snapshot = Snapshot.createSnapshot(buffer); + logger.info("start createSnapshot: " + heapDumpFile.getName()); + + deduplicateGcRoots(snapshot); + logger.info("deduplicateGcRoots: " + heapDumpFile.getName()); + + snapshot.computeDominators(); + logger.info("compute dominators finished: " + heapDumpFile.getName()); + + List dominatorList = snapshot.getReachableInstances(); + if (dominatorList.isEmpty()) { + logger.warn("no gReachable Instances: in file: {}", heapDumpFile.getName()); + return; + } + dominatorList.sort((instance1, instance2) -> -Long.compare(instance1.getTotalRetainedSize(), instance2.getTotalRetainedSize())); + logger.info("sortByRetainedSize finish start listing dominatorList: {}, in file: {}", dominatorList.size(), heapDumpFile.getName()); + + Iterator instanceIterator = dominatorList.iterator(); + + int index = 0; + while (instanceIterator.hasNext()) { + Instance instance = instanceIterator.next(); + if (instance.getClassObj() == null) { + continue; + } + processInstance(index++, instance); + } + + // sort all extracted object info by retained size + for (Map.Entry entry : extractorMap.entrySet()) { + entry.getValue().onExtractComplete(); + } + } + + /** + * Pruning duplicates reduces memory pressure from hprof bloat added in Marshmallow. + */ + private void deduplicateGcRoots(Snapshot snapshot) { + // THashMap has a smaller memory footprint than HashMap. + final THashMap uniqueRootMap = new THashMap<>(); + + final Collection gcRoots = snapshot.getGCRoots(); + for (RootObj root : gcRoots) { + String key = generateRootKey(root); + if (!uniqueRootMap.containsKey(key)) { + uniqueRootMap.put(key, root); + } + } + + // Repopulate snapshot with unique GC roots. + gcRoots.clear(); + uniqueRootMap.forEach(new TObjectProcedure() { + @Override + public boolean execute(String key) { + return gcRoots.add(uniqueRootMap.get(key)); + } + }); + } + + private void processInstance(int index, Instance instance) { + for (Map.Entry entry : extractorMap.entrySet()) { + entry.getValue().onExtractInfo(index, instance); + } + } + + private static void findRelatedInstances(Instance instance, List founds, Instance visited, List fieldNames) { + if (instance == null) { + return; + } + if (visited != null) { + if (instance instanceof ClassInstance) { + ClassInstance classInstance = (ClassInstance) instance; + List values = classInstance.getValues(); + for (ClassInstance.FieldValue value : values) { + if (Objects.equals(value.getField().getType(), Type.OBJECT)) { + if (Objects.equals(value.getValue(), visited)) { + founds.add(instance); + fieldNames.add(value.getField().getName()); + break; + } + } + } + } else if (instance instanceof ClassObj) { + Map staticFieldValues = ((ClassObj) instance).getStaticFieldValues(); + for (Map.Entry fieldObjectEntry : staticFieldValues.entrySet()) { + Field key = fieldObjectEntry.getKey(); + if (Objects.equals(key.getType(), Type.OBJECT)) { + if (Objects.equals(staticFieldValues.get(key), visited)) { + founds.add(instance); + fieldNames.add(key.getName()); + break; + } + } + } + } else if (instance instanceof ArrayInstance) { + final ArrayInstance arrayInstance = (ArrayInstance) instance; + int i = 0; + for (Object object : arrayInstance.getValues()) { + if (Objects.equals(object, visited)) { + founds.add(instance); + fieldNames.add(i + ""); + break; + } + i++; + } + } + } + findRelatedInstances(instance.getNextInstanceToGcRoot(), founds, instance, fieldNames); + } + + private static List getPathToGCRoot(Instance instance) { + List gcRoots = new ArrayList<>(); + List founds = new ArrayList<>(); + List fieldNames = new ArrayList<>(); + findRelatedInstances(instance, founds, null, fieldNames); + for (int i = founds.size() - 1; i >= 0; i--) { + Instance foundInstance = founds.get(i); + String fieldName = fieldNames.get(i); + ClassObj classObj = foundInstance.getClassObj(); + if (classObj == null) { + if (!(foundInstance instanceof ClassObj)) { + continue; + } + classObj = (ClassObj) foundInstance; + } + String message = ""; + message += String.format("%s %s.%s", + getSizeStringWithTagIfLarge(foundInstance.getTotalRetainedSize(), ObjectInfo.SIZE_THRESHOLD), + classObj.getClassName(), fieldName); + gcRoots.add(new FieldChain(message, foundInstance)); + + } + return gcRoots; + } + + public static String generateFieldChainString(Instance instance, String lineEnding) { + if (instance == null) { + return ""; + } + + StringBuilder builder = new StringBuilder(); + List gcRoots = getPathToGCRoot(instance); + for (FieldChain fieldChain : gcRoots) { + String className; + if (fieldChain.instance.getClassObj() == null) { + className = ((ClassObj) fieldChain.instance).getClassName(); + } else { + className = fieldChain.instance.getClassObj().getClassName(); + } + if (className != null && (className.startsWith("android.graphics.Bitmap") || className.startsWith("java.util.HashMap$Node"))) { + continue; + } + if (className != null && className.contains(".launcher")) { + builder.append("").append("↱ ").append(fieldChain.message).append("").append(lineEnding); + } else { + builder.append("↱ ").append(fieldChain.message).append(lineEnding); + } + } + + + if (instance.getClassObj() != null) { + builder.append(instance.getClassObj().getClassName()).append(" "); + } + + List fieldChains = generateDominatedFieldChain(instance, 0, new HashSet()); + if (!fieldChains.isEmpty()) { + builder.append(lineEnding); + + for (FieldChain fieldChain : fieldChains) { + String className = fieldChain.instance.getClassObj().getClassName(); + if (className != null && (className.startsWith("android.graphics.Bitmap") || className.startsWith("java.util.HashMap$Node"))) { + continue; + } + if (className != null && className.contains(".launcher")) { + builder.append("").append("↳ ").append(fieldChain.message).append("").append(lineEnding); + } else { + builder.append("↳ ").append(fieldChain.message).append(lineEnding); + } + } + } + builder.append(lineEnding); + return builder.toString(); + } + + private static List generateDominatedFieldChain(Instance instance, int indent, Set chainInstanceSet) { + final List fieldChains = new ArrayList<>(); + if (indent > MAX_FIELD_CHAIN_DEPTH) { + // Limit recursion depth + return fieldChains; + } + + if (instance instanceof ArrayInstance) { + Instance maxRetainedSize = null; + int maxIndex = -1; + final ArrayInstance arrayInstance = (ArrayInstance) instance; + int i = 0; + for (Object object : arrayInstance.getValues()) { + if (object == null) { + continue; + } + + if (object instanceof Instance) { + Instance instance1 = (Instance) object; + + if (chainInstanceSet.contains(instance1.getId())) { + continue; + } + + if (maxRetainedSize == null + || instance1.getTotalRetainedSize() > maxRetainedSize.getTotalRetainedSize()) { + maxRetainedSize = instance1; + maxIndex = i; + } + } + ++i; + } + + if (maxRetainedSize != null) { + String message = ""; + message += String.format("%s %s%d", + getSizeStringWithTagIfLarge(instance.getTotalRetainedSize(), ObjectInfo.SIZE_THRESHOLD), + instance.getClassObj().getClassName(), + maxIndex); + logger.debug(message); + fieldChains.add(new FieldChain(message, instance)); + chainInstanceSet.add(maxRetainedSize.getId()); + fieldChains.addAll(generateDominatedFieldChain(maxRetainedSize, indent + 1, chainInstanceSet)); + } + } else if (instance instanceof ClassInstance) { + ClassInstance classInstance = (ClassInstance) instance; + List fieldValues = classInstance.getValues(); + Instance maxRetainedSize = null; + ClassInstance.FieldValue maxFieldValue = null; + for (ClassInstance.FieldValue fieldValue : fieldValues) { + Object value = fieldValue.getValue(); + if (value == null) { + continue; + } + + if (value instanceof Instance) { + Instance instance1 = (Instance) value; + + if (chainInstanceSet.contains(instance1.getId())) { + continue; + } + + if (maxRetainedSize == null + || instance1.getTotalRetainedSize() > maxRetainedSize.getTotalRetainedSize()) { + maxRetainedSize = instance1; + maxFieldValue = fieldValue; + } + } else { + logger.debug("ignore unknown field value type " + value.getClass().getName()); + } + } + + if (maxRetainedSize != null) { + String message = ""; + message += String.format("%s %s.%s", + getSizeStringWithTagIfLarge(instance.getTotalRetainedSize(), ObjectInfo.SIZE_THRESHOLD), + instance.getClassObj().getClassName(), + maxFieldValue.getField().getName()); + logger.debug(message); + fieldChains.add(new FieldChain(message, instance)); + chainInstanceSet.add(maxRetainedSize.getId()); + fieldChains.addAll(generateDominatedFieldChain(maxRetainedSize, indent + 1, chainInstanceSet)); + } + } else { + logger.debug("ignore unknown instance type " + instance.getClass().getName()); + } + + return fieldChains; + } + + private static class FieldChain { + public final String message; + public final Instance instance; + + FieldChain(String message, Instance instance) { + this.message = message; + this.instance = instance; + } + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/hprof/ObjectInfo.java b/common/src/main/java/com/microsoft/hydralab/performance/hprof/ObjectInfo.java new file mode 100644 index 000000000..fe6bfe974 --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/hprof/ObjectInfo.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.hprof; + +import com.squareup.haha.perflib.ClassObj; +import com.squareup.haha.perflib.Instance; + +import java.io.Serializable; + +import static com.microsoft.hydralab.common.util.FileUtil.getSizeStringWithTagIfLarge; + +public class ObjectInfo implements Serializable { + public static final int SIZE_THRESHOLD = 500 * 1024; + + public int index; + public int distanceToRoot; + public String fieldName; + public Instance instance; + public Instance firstLevelLauncherRef; + public long nativeSize; + public long retainedSize; + public long size; + public boolean isStaticMember; + public long id; + public long uniqueId; + public String className; + + public String getFieldChain(String lineEnding) { + if (className != null && className.startsWith("dalvik.system.PathClassLoader")) { + return null; + } else { + return HeapProfProcessor.generateFieldChainString(instance, lineEnding); + } + } + + protected String getRefClassName() { + if (firstLevelLauncherRef == null) { + return "null"; + } + ClassObj classObj = firstLevelLauncherRef.getClassObj(); + if (classObj == null) { + if (firstLevelLauncherRef instanceof ClassObj) { + return ((ClassObj) firstLevelLauncherRef).getClassName(); + } + return "null"; + } + return classObj.getClassName(); + } + + protected String getSizeInfo() { + String retainedSizeString = "RetainedSize: " + getSizeStringWithTagIfLarge(retainedSize, SIZE_THRESHOLD); + String nativeSizeString = ""; + if (nativeSize != 0) { + nativeSizeString = ",  NativeSize: " + getSizeStringWithTagIfLarge(nativeSize, SIZE_THRESHOLD); + } + return retainedSizeString + nativeSizeString; + } + + public String getFieldChainString() { + String fieldChain = getFieldChain("
"); + if (fieldChain == null) { + return null; + } + return "" + fieldChain + ""; + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/hprof/TopObjectInfoExtractor.java b/common/src/main/java/com/microsoft/hydralab/performance/hprof/TopObjectInfoExtractor.java new file mode 100644 index 000000000..eebb5911f --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/hprof/TopObjectInfoExtractor.java @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.hprof; + +import com.squareup.haha.perflib.ClassInstance; +import com.squareup.haha.perflib.ClassObj; +import com.squareup.haha.perflib.Instance; + +public class TopObjectInfoExtractor extends Extractor { + private final int count; + + public TopObjectInfoExtractor(int count) { + this.count = count; + } + + @Override + public ObjectInfo extractInstanceInfo(int retainedSizeRanking, Instance instance) { + if (retainedSizeRanking > count) { + return null; + } + + ClassObj classObj = instance.getClassObj(); + String className = classObj.getClassName(); + if (className.equals("android.graphics.Bitmap")) { + return extractBitmapInfo(retainedSizeRanking, instance); + } + + if (instance instanceof ClassInstance) { + ObjectInfo objectInfo = new ObjectInfo(); + mapSetBaseAttr(instance, objectInfo); + return objectInfo; + } + return null; + } + + @Override + public String getType() { + return "top" + count; + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidMemoryHprofInspector.java b/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidMemoryHprofInspector.java new file mode 100644 index 000000000..cef7fe91b --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidMemoryHprofInspector.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.inspectors; + +import com.microsoft.hydralab.common.util.ShellUtils; +import com.microsoft.hydralab.common.util.TimeUtils; +import com.microsoft.hydralab.performance.PerformanceInspection; +import com.microsoft.hydralab.performance.PerformanceInspectionResult; +import com.microsoft.hydralab.performance.PerformanceInspector; +import org.slf4j.Logger; +import org.springframework.util.Assert; + +import java.io.File; + +public class AndroidMemoryHprofInspector implements PerformanceInspector { + + private static final String RAW_RESULT_FILE_NAME_FORMAT = "memory_%s.hprof"; + private static final String HPROF_FILE_PREFIX = "/data/local/tmp/"; + + + @Override + public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection, Logger logger) { + + File rawResultFolder = new File(performanceInspection.resultFolder, performanceInspection.appId); + Assert.isTrue(rawResultFolder.exists() || rawResultFolder.mkdir(), "rawResultFolder.mkdirs() failed in" + rawResultFolder.getAbsolutePath()); + String tmpTime = TimeUtils.getTimestampForFilename(); + String hprofFileName = String.format(RAW_RESULT_FILE_NAME_FORMAT, tmpTime); + File rawResultFile = new File(rawResultFolder, + hprofFileName); + String sdHprofFilePath = HPROF_FILE_PREFIX + hprofFileName; + String dumpCommand = String.format(getMemHprofCommand(), performanceInspection.deviceIdentifier, performanceInspection.appId, sdHprofFilePath); + if (!isDebuggable(performanceInspection.deviceIdentifier, performanceInspection.appId, logger)) { + return new PerformanceInspectionResult(null, performanceInspection); + } else { + Process process = ShellUtils.execLocalCommand(dumpCommand, false, logger); + if (process != null) { + try { + int ret = process.waitFor(); + if (ret == 0) { + Process pullProcess = ShellUtils.execLocalCommand("adb pull " + sdHprofFilePath + " " + rawResultFile.getAbsolutePath(), false, logger); + if (pullProcess != null) { + ret = pullProcess.waitFor(); + if (ret == 0) { + return new PerformanceInspectionResult(rawResultFile, performanceInspection); + } + } + } + } catch (InterruptedException e) { + logger.error(e.getMessage(), e); + return new PerformanceInspectionResult(null, performanceInspection); + } + } + + + } + return new PerformanceInspectionResult(null, performanceInspection); + } + + private String getMemHprofCommand() { + return "adb -s %s shell am dumpheap %s %s"; + } + + + private boolean isDebuggable(String deviceId, String packageName, Logger logger) { + String cmd = String.format("adb -s %s shell dumpsys package %s | findstr flags", deviceId, packageName); + String ret = ShellUtils.execLocalCommandWithResult(cmd, logger); + return ret != null && ret.contains("DEBUGGABLE"); + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidMemoryInfoInspector.java b/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidMemoryInfoInspector.java index a06086faf..5465dcd33 100644 --- a/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidMemoryInfoInspector.java +++ b/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidMemoryInfoInspector.java @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + package com.microsoft.hydralab.performance.inspectors; import com.microsoft.hydralab.common.util.ShellUtils; diff --git a/common/src/main/java/com/microsoft/hydralab/performance/parsers/AndroidMemoryHprofResultParser.java b/common/src/main/java/com/microsoft/hydralab/performance/parsers/AndroidMemoryHprofResultParser.java new file mode 100644 index 000000000..edfdbc457 --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/parsers/AndroidMemoryHprofResultParser.java @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.hydralab.performance.parsers; + +import com.microsoft.hydralab.performance.PerformanceInspectionResult; +import com.microsoft.hydralab.performance.PerformanceResultParser; +import com.microsoft.hydralab.performance.PerformanceTestResult; +import com.microsoft.hydralab.performance.entity.AndroidHprofMemoryInfo; +import com.microsoft.hydralab.performance.hprof.BitmapInfoExtractor; +import com.microsoft.hydralab.performance.hprof.HeapProfProcessor; +import com.microsoft.hydralab.performance.hprof.TopObjectInfoExtractor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +public class AndroidMemoryHprofResultParser implements PerformanceResultParser { + private static final int MD_OBJECT_REPORT_COUNT = 50; + + @Override + public PerformanceTestResult parse(PerformanceTestResult performanceTestResult, Logger logger) { + if (performanceTestResult == null || performanceTestResult.performanceInspectionResults == null + || performanceTestResult.performanceInspectionResults.isEmpty()) { + return null; + } + List inspectionResults = performanceTestResult.performanceInspectionResults; + for (PerformanceInspectionResult inspectionResult : inspectionResults) { + File hprofFile = inspectionResult.rawResultFile; + if (hprofFile == null) { + return performanceTestResult; + } + inspectionResult.parsedData = buildAndroidHprofMemoryInfo(hprofFile, inspectionResult.inspection.appId, inspectionResult.inspection.description, + inspectionResult.timestamp, logger); + } + + return performanceTestResult; + } + + + + private AndroidHprofMemoryInfo buildAndroidHprofMemoryInfo(File dumpFile, String packageName, String description, long timeStamp, Logger logger) { + + HeapProfProcessor profProcessor = new HeapProfProcessor(dumpFile); + + BitmapInfoExtractor bitmapInfoExtractor = new BitmapInfoExtractor(); + profProcessor.registerExtractor(bitmapInfoExtractor); + + TopObjectInfoExtractor topObjectInfoExtractor = new TopObjectInfoExtractor(MD_OBJECT_REPORT_COUNT); + profProcessor.registerExtractor(topObjectInfoExtractor); + try { + profProcessor.loadAndExtract(); + } catch (IOException e) { + logger.error("parseHprofFile", e); + return null; + } + AndroidHprofMemoryInfo info = new AndroidHprofMemoryInfo(); + info.setAppPackageName(packageName); + info.setDescription(description); + info.setTimeStamp(timeStamp); + info.setBitmapInfoList(bitmapInfoExtractor.getResultList()); + info.setTopObjectList(topObjectInfoExtractor.getResultList()); + return info; + + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/parsers/AndroidMemoryInfoResultParser.java b/common/src/main/java/com/microsoft/hydralab/performance/parsers/AndroidMemoryInfoResultParser.java index 1d5c35b86..8cf1416d1 100644 --- a/common/src/main/java/com/microsoft/hydralab/performance/parsers/AndroidMemoryInfoResultParser.java +++ b/common/src/main/java/com/microsoft/hydralab/performance/parsers/AndroidMemoryInfoResultParser.java @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + package com.microsoft.hydralab.performance.parsers; import com.google.common.base.Strings; diff --git a/sdk/build.gradle b/sdk/build.gradle index 73a3f4fe0..b2a5c0e00 100644 --- a/sdk/build.gradle +++ b/sdk/build.gradle @@ -11,6 +11,7 @@ repositories { } dependencies { + implementation 'org.jetbrains:annotations:21.0.1' compile 'ch.qos.logback:logback-classic:1.2.3' compile 'org.slf4j:slf4j-api:1.7.30' } diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java index d36e8477e..688e618e8 100644 --- a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java @@ -3,6 +3,8 @@ package com.microsoft.hydralab.performance; +import org.jetbrains.annotations.Nullable; + import java.io.File; public class PerformanceInspectionResult { @@ -14,6 +16,7 @@ public class PerformanceInspectionResult { @SuppressWarnings("visibilitymodifier") public PerformanceInspection inspection; @SuppressWarnings("visibilitymodifier") + @Nullable public File rawResultFile; @SuppressWarnings("visibilitymodifier") // TODO: restrict the size of it. diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceResultParser.java b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceResultParser.java index 4ebba357e..7d519cd4e 100644 --- a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceResultParser.java +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceResultParser.java @@ -5,8 +5,11 @@ import org.slf4j.Logger; +import org.jetbrains.annotations.Nullable; + public interface PerformanceResultParser { - PerformanceTestResult parse(PerformanceTestResult performanceTestResult, Logger logger); + @Nullable + PerformanceTestResult parse(@Nullable PerformanceTestResult performanceTestResult, Logger logger); enum PerformanceResultParserType { PARSER_ANDROID_MEMORY_DUMP,