feat: Add bloat analysis UI and improve singleton detection (#18)

Author: rcd47

x2-data-explorer/docs/bloat-analysis-tab.png

@@ -18,7 +18,7 @@ Multiple files can be loaded at the same time. Each file will be in a separate t
 
 Save files are produced when you save a regular game. History files are produced when you save a ladder game. Save and history files have the same format and work the same way, except that save files have an additional header with some extra information. Both file types contain the entire game state history since the last time the history was archived (which occurs during each map transition).
 
-After loading one of these files, the tab will contain three sub-tabs: the <<general-tab>>, the <<frames-tab>>, and the <<problems-tab>>.
+After loading one of these files, the tab will contain four sub-tabs: the <<general-tab>>, the <<frames-tab>>, the <<bloat-tab>>, and the <<problems-tab>>.
 
 [#general-tab]
 === General Tab
@@ -29,7 +29,7 @@ For a save file, the General tab contains four sections:
 
 * Top left: information from the save file header. This can tell you things like: when the save file was created, whether it was created in tactical or strategy, and which version of the game created it.
 * Top right: a list of mods and DLCs that were active when the save was created.
-* Bottom left: a list of singleton XComGameStateObjects. Singleton objects are objects whose class has `bSingletonStateType` set to true. Historical versions of these objects are not serialized, so unlike other objects, there is no way to know what a singleton object looked like at each frame in the history. Fortunately, singleton objects are very rare. XComGameState_Analytics is the only such class in the base game.
+* Bottom left: a list of singleton XComGameStateObjects. Singleton objects are objects whose class has `bSingletonStateType` set to true. Historical versions of these objects are not serialized, except in strategy saves, where the archive frame holds the original state of the object. So unlike other objects, there is no way to know what a singleton object looked like at each frame in the history. Fortunately, singleton objects are very rare. XComGameState_Analytics is the only such class in the base game.
 * Bottom right: the object properties tree, populated when you click on any singleton object.
 
 For a history file, the General tab is mostly the same, except the list of mods and DLCs is removed, and the header information only contains a few fields that are available from XComGameStateHistory.
@@ -83,6 +83,20 @@ IMPORTANT: When writing expressions to match fields that are Unreal names, remem
 
 The frames and objects tables have a summary column that provides a short description of what that frame or object is about. The summaries are powered by Groovy scripts. Default scripts are included in the application, but can be modified if you like. In the Preferences menu at the top of the screen, click the State Object Summary Script or Context Summary Script menu items to edit the scripts.
 
+[#bloat-tab]
+=== Bloat Analysis Tab
+
+image::bloat-analysis-tab.png[]
+
+The Bloat Analysis tab helps you figure out what's using the most space in your save files. It has six sub-tabs:
+
+. *Object Class Stats* provides summary statistics for objects that are subclasses of `XComGameState_BaseObject`. It shows the number of objects, min/max/average number of deltas per object (i.e. the number of times an object was changed), and the min/max/average size of each delta.
+. *Context Class Stats* provides summary statistics for objects that are subclasses of `XComGameStateContext`. It shows the number of frames that used a context of each class, and the min/max/average/total bytes used by those contexts.
+. *Largest Delta Objects* lists the top 500 largest delta objects in the file. The object ID and frame number are provided so you can switch to the Frames tab to get a better idea of what was happening in that frame.
+. *Largest Full Objects* lists the top 500 largest full (non-delta) objects in the file. The object ID and frame number are provided so you can switch to the Frames tab to get a better idea of what was happening in that frame.
+. *Largest Contexts* lists the top 500 largest contexts in the file. The frame number is provided so you can switch to the Frames tab to get a better idea of what was happening in that frame.
+. *Singletons* shows the size of all singleton state objects in the file.
+
 [#problems-tab]
 === Problems Tab
 

x2-data-explorer/docs/save-general-tab.PNG

@@ -11,12 +11,13 @@
 
 import groovy.lang.Script;
 
-public class GameStateContext {
+public class GameStateContext implements ISizedObject {
 
 	private static final UnrealName INTERRUPTION_HISTORY_INDEX = new UnrealName("InterruptionHistoryIndex");
 	private static final UnrealName HISTORY_INDEX_INTERRUPTED_BY_SELF = new UnrealName("HistoryIndexInterruptedBySelf");
 	private static final UnrealName INTERRUPTION_STATUS = new UnrealName("InterruptionStatus");
 
+	private final int sizeInFile;
 	private final UnrealName type;
 	private final Map<UnrealName, NonVersionedField> fields;
 	private final HistoryFrame frame;
@@ -26,8 +27,9 @@ public class GameStateContext {
 	private final HistoryFrame interruptedByThis;
 	private HistoryFrame resumedBy;
 
-	public GameStateContext(GenericObject object, HistoryFrame frame, Map<Integer, HistoryFrame> frames, Script summarizer,
+	public GameStateContext(int sizeInFile, GenericObject object, HistoryFrame frame, Map<Integer, HistoryFrame> frames, Script summarizer,
 			List<HistoryFileProblem> problemsDetected) {
+		this.sizeInFile = sizeInFile;
 		this.frame = frame;
 
 		type = object.type;
@@ -68,6 +70,11 @@ public Object propertyMissing(String name) {
 		return fields.get(new UnrealName(name));
 	}
 
+	@Override
+	public int getSizeInFile() {
+		return sizeInFile;
+	}
+
 	public UnrealName getType() {
 		return type;
 	}

x2-data-explorer/docs/user-guide.adoc

@@ -13,11 +13,12 @@
 import groovy.lang.Script;
 import javafx.scene.control.TreeItem;
 
-public class GameStateObject {
+public class GameStateObject implements ISizedObject {
 
 	private static final UnrealName OBJECT_ID = new UnrealName("ObjectID");
 	private static final UnrealName REMOVED = new UnrealName("bRemoved");
 
+	private final int sizeInFile;
 	private final int objectId;
 	private final boolean removed; // note that it is possible for an object to be added and removed in the same state
 	private final UnrealName type;
@@ -27,8 +28,9 @@ public class GameStateObject {
 	private final GameStateObject previousVersion;
 	private GameStateObject nextVersion;
 
-	public GameStateObject(Map<Integer, GameStateObject> stateObjects, GenericObject currentVersion, HistoryFrame frame, Script summarizer,
-			List<HistoryFileProblem> problemsDetected) {
+	public GameStateObject(int sizeInFile, Map<Integer, GameStateObject> stateObjects, GenericObject currentVersion,
+			HistoryFrame frame, Script summarizer, List<HistoryFileProblem> problemsDetected) {
+		this.sizeInFile = sizeInFile;
 		this.frame = frame;
 
 		objectId = (int) currentVersion.properties.get(OBJECT_ID);
@@ -71,6 +73,11 @@ public TreeItem<GameStateObjectFieldTreeNode> getFieldsAsTreeNode(boolean onlyMo
 		return root;
 	}
 
+	@Override
+	public int getSizeInFile() {
+		return sizeInFile;
+	}
+
 	public int getObjectId() {
 		return objectId;
 	}

x2-data-explorer/src/main/java/com/github/rcd47/x2data/explorer/file/GameStateContext.java

@@ -6,11 +6,12 @@
 import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.DoubleConsumer;
 
@@ -38,38 +39,65 @@ public HistoryFile read(FileChannel in, DoubleConsumer progressPercentCallback,
 			reader.decompress(in, decompressedIn);
 			progressTextCallback.accept("Building index");
 			try (var historyIndex = reader.buildIndex(decompressedIn)) {
-				Map<Integer, HistoryFrame> frames = new HashMap<>();
+				XComGameStateHistory history = historyIndex.mapObject(historyIndex.getEntry(0), null, NullXComObjectReferenceResolver.INSTANCE);
+				var frameRefs = history.History;
+				var currentFrameNum = history.NumArchivedFrames + 1;
+				if (!historyIndex.isCreatedByWOTC()) {
+					// before WOTC, NumArchivedFrames did not exist and archived frames were represented by -1 in the History array
+					for (int i = frameRefs.size() - 1; i >= 0; i--) {
+						if (frameRefs.get(i).index() == -1) {
+							frameRefs = frameRefs.subList(i + 1, frameRefs.size());
+							currentFrameNum = i + 2;
+							break;
+						}
+					}
+				}
+				
+				// first pass to parse the state and detect singletons
+				var numFrames = frameRefs.size();
+				var rawFrames = new XComGameState[numFrames];
+				var seenObjectIndexes = new HashSet<Integer>();
+				var detectedSingletonTypes = new HashSet<UnrealName>();
+				for (int i = 0; i < numFrames; i++) {
+					progressTextCallback.accept("Parsing history frame " + currentFrameNum++);
+					XComGameState rawFrame = historyIndex.mapObject(
+							historyIndex.getEntry(frameRefs.get(i).index()), null, NullXComObjectReferenceResolver.INSTANCE);
+					rawFrames[i] = rawFrame;
+					for (var objRef : rawFrame.GameStates) {
+						if (!seenObjectIndexes.add(objRef.index())) {
+							// multiple frames pointing to same object index, so class must be a singleton
+							// note that in strategy saves, two versions of a singleton are written
+							// the first frame (archive frame) points to one version
+							// all other frames point to the other version
+							// this does not happen for tactical saves, where all frames point to a single version
+							detectedSingletonTypes.add(historyIndex.getEntry(objRef.index()).getType());
+						}
+					}
+				}
+				
+				// second pass to parse the state objects
+				Map<Integer, HistoryFrame> frames = new LinkedHashMap<>();
 				Map<Integer, GenericObject> parsedObjects = new HashMap<>();
 				Map<Integer, GameStateObject> stateObjects = new HashMap<>();
-				Set<X2HistoryIndexEntry> singletonStates = new HashSet<>();
+				Map<X2HistoryIndexEntry, Integer> singletonStates = new HashMap<>();
 				List<HistoryFileProblem> problemsDetected = new ArrayList<>();
 				var contextSummarizer = ScriptPreferences.CONTEXT_SUMMARY.getExecutable();
 				var objectSummarizer = ScriptPreferences.STATE_OBJECT_SUMMARY.getExecutable();
-				
-				XComGameStateHistory history = historyIndex.mapObject(historyIndex.getEntry(0), null, NullXComObjectReferenceResolver.INSTANCE);
-				int numFrames = history.History.size();
-				boolean foundFirstFrame = historyIndex.isCreatedByWOTC();
 				for (int i = 0; i < numFrames; i++) {
-					var frameRef = history.History.get(i);
-					if (!foundFirstFrame && frameRef.index() == -1) {
-						// before WOTC, NumArchivedFrames did not exist and archived frames were represented by -1 in the History array
-						continue;
-					}
-					XComGameState rawFrame = historyIndex.mapObject(
-							historyIndex.getEntry(frameRef.index()), null, NullXComObjectReferenceResolver.INSTANCE);
+					XComGameState rawFrame = rawFrames[i];
 					var parsedFrame = new HistoryFrame(rawFrame.HistoryIndex, rawFrame.TimeStamp);
-					progressTextCallback.accept("Parsing history frame " + rawFrame.HistoryIndex);
+					progressTextCallback.accept("Parsing objects for history frame " + rawFrame.HistoryIndex);
 
 					var contextEntry = historyIndex.getEntry(rawFrame.StateChangeContext.index());
 					var contextVisitor = new GenericObjectVisitor(null);
 					historyIndex.parseObject(contextEntry, contextVisitor);
 					var parsedContext = new GameStateContext(
-							contextVisitor.getRootObject(), parsedFrame, frames, contextSummarizer, problemsDetected);
+							contextEntry.getLength(), contextVisitor.getRootObject(), parsedFrame, frames, contextSummarizer, problemsDetected);
 
 					for (var stateObjectRef : rawFrame.GameStates) {
 						var stateObjectEntry = historyIndex.getEntry(stateObjectRef.index());
-						if (stateObjectEntry.isSingletonState()) {
-							singletonStates.add(stateObjectEntry);
+						if (detectedSingletonTypes.contains(stateObjectEntry.getType())) {
+							singletonStates.putIfAbsent(stateObjectEntry, rawFrame.HistoryIndex);
 							continue;
 						}
 
@@ -94,7 +122,7 @@ public HistoryFile read(FileChannel in, DoubleConsumer progressPercentCallback,
 											stateObject.properties.get(PREV_FRAME_HIST_INDEX))));
 						} else {
 							parsedObjects.put(stateObjectRef.index(), stateObject);
-							new GameStateObject(stateObjects, stateObject, parsedFrame, objectSummarizer, problemsDetected); // adds itself to the map
+							new GameStateObject(stateObjectEntry.getLength(), stateObjects, stateObject, parsedFrame, objectSummarizer, problemsDetected); // adds itself to the map
 						}
 					}
 
@@ -103,22 +131,27 @@ public HistoryFile read(FileChannel in, DoubleConsumer progressPercentCallback,
 					progressPercentCallback.accept(((double) i + 1) / numFrames);
 				}
 
+				progressTextCallback.accept("Parsing singletons");
 				var singletons = singletonStates
+						.entrySet()
 						.stream()
-						.map(s -> {
+						.map(entry -> {
+							var key = entry.getKey();
 							var stateObjectVisitor = new GenericObjectVisitor(null);
 							try {
-								historyIndex.parseObject(s, stateObjectVisitor);
+								historyIndex.parseObject(key, stateObjectVisitor);
 							} catch (IOException e) {
 								// should never happen
 								throw new UncheckedIOException(e);
 							}
-							return new HistorySingletonObject(stateObjectVisitor.getRootObject());
+							return new HistorySingletonObject(key.getLength(), entry.getValue(), stateObjectVisitor.getRootObject());
 						})
-						.sorted((a, b) -> a.getType().compareTo(b.getType()))
+						.sorted(Comparator
+								.<HistorySingletonObject, UnrealName>comparing(s -> s.getType())
+								.thenComparingInt(s -> s.getFirstFrame()))
 						.toList();
 
-				return new HistoryFile(history, frames.values().stream().sorted().toList(), singletons, problemsDetected);
+				return new HistoryFile(history, List.copyOf(frames.values()), singletons, problemsDetected);
 			}
 		} finally {
 			Files.deleteIfExists(decompressedFile);

x2-data-explorer/src/main/java/com/github/rcd47/x2data/explorer/file/GameStateObject.java

@@ -5,21 +5,34 @@
 
 import com.github.rcd47.x2data.lib.unreal.mappings.UnrealName;
 
-public class HistorySingletonObject {
+public class HistorySingletonObject implements ISizedObject {
 
 	private static final UnrealName OBJECT_ID = new UnrealName("ObjectID");
 
+	private final int sizeInFile;
+	private final int firstFrame;
 	private final int objectId;
 	private final UnrealName type;
 	private final Map<UnrealName, NonVersionedField> fields;
 
-	public HistorySingletonObject(GenericObject object) {
+	public HistorySingletonObject(int sizeInFile, int firstFrame, GenericObject object) {
+		this.sizeInFile = sizeInFile;
+		this.firstFrame = firstFrame;
 		objectId = (int) object.properties.get(OBJECT_ID);
 		type = object.type;
 		fields = new HashMap<>();
 		object.properties.forEach((k, v) -> fields.put(k, new NonVersionedField(v)));
 	}
 
+	@Override
+	public int getSizeInFile() {
+		return sizeInFile;
+	}
+
+	public int getFirstFrame() {
+		return firstFrame;
+	}
+
 	public int getObjectId() {
 		return objectId;
 	}

x2-data-explorer/src/main/java/com/github/rcd47/x2data/explorer/file/HistoryFileReader.java

@@ -0,0 +1,7 @@
+package com.github.rcd47.x2data.explorer.file;
+
+public interface ISizedObject {
+	
+	int getSizeInFile();
+	
+}