Skip to content

Commit f6d3b58

Browse files
Pranshu-SPranshu-S
committed
Optimize NodeIndicesStats output behind flag
Signed-off-by: Pranshu Shukla <[email protected]>
1 parent 93d507a commit f6d3b58

File tree

7 files changed

+204
-17
lines changed

7 files changed

+204
-17
lines changed

‎server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java

+45
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010

1111
import org.opensearch.ExceptionsHelper;
1212
import org.opensearch.action.DocWriteResponse;
13+
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
14+
import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
1315
import org.opensearch.action.bulk.BulkItemResponse;
1416
import org.opensearch.action.bulk.BulkRequest;
1517
import org.opensearch.action.bulk.BulkResponse;
@@ -19,6 +21,7 @@
1921
import org.opensearch.action.index.IndexResponse;
2022
import org.opensearch.action.update.UpdateRequest;
2123
import org.opensearch.action.update.UpdateResponse;
24+
import org.opensearch.cluster.ClusterState;
2225
import org.opensearch.index.IndexNotFoundException;
2326
import org.opensearch.index.engine.DocumentMissingException;
2427
import org.opensearch.index.engine.VersionConflictEngineException;
@@ -31,6 +34,7 @@
3134
import java.util.Arrays;
3235
import java.util.Comparator;
3336
import java.util.Map;
37+
import java.util.ArrayList;
3438
import java.util.concurrent.atomic.AtomicLong;
3539

3640
import static java.util.Collections.singletonMap;
@@ -242,6 +246,47 @@ public void testNodeIndicesStatsDocStatusStatsCreateDeleteUpdate() {
242246
}
243247
}
244248
}
249+
public void testNodeIndicesStatsOptimisedResponse() {
250+
internalCluster().startNode();
251+
ensureGreen();
252+
String indexName = "test1";
253+
index(indexName, "type", "1", "f", "f");
254+
refresh();
255+
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
256+
257+
NodesStatsResponse response = client().admin().cluster().prepareNodesStats().get();
258+
response.getNodes().forEach(nodeStats -> {
259+
assertNotNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex()));
260+
assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex()));
261+
});
262+
CommonStatsFlags commonStatsFlags = new CommonStatsFlags();
263+
commonStatsFlags.optimizeNodeIndicesStatsOnLevel(true);
264+
response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get();
265+
response.getNodes().forEach(nodeStats -> {
266+
assertNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex()));
267+
assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex()));
268+
269+
});
270+
ArrayList<String> level_arg = new ArrayList<>();
271+
level_arg.add("indices");
272+
commonStatsFlags.setLevels(level_arg.toArray(new String[0]));
273+
response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get();
274+
response.getNodes().forEach(nodeStats -> {
275+
assertNotNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex()));
276+
assertNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex()));
277+
});
278+
279+
level_arg.clear();
280+
level_arg.add("shards");
281+
commonStatsFlags.setLevels(level_arg.toArray(new String[0]));
282+
response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get();
283+
response.getNodes().forEach(nodeStats -> {
284+
assertNotNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex()));
285+
assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex()));
286+
});
287+
}
288+
289+
245290

246291
private void assertDocStatusStats() {
247292
DocStatusStats docStatusStats = client().admin()

‎server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java

+10
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,16 @@ public NodesStatsRequest indices(boolean indices) {
127127
return this;
128128
}
129129

130+
/**
131+
* Use Optimized Response filtered based on level
132+
*/
133+
public NodesStatsRequest useOptimizedNodeIndicesStats(boolean useOptimizedNodeIndicesStats){
134+
if (this.indices!=null) {
135+
this.indices.optimizeNodeIndicesStatsOnLevel(true);
136+
}
137+
return this;
138+
}
139+
130140
/**
131141
* Get the names of requested metrics, excluding indices, which are
132142
* handled separately.

‎server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java

+16
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ public class CommonStatsFlags implements Writeable, Cloneable {
6767
// Used for metric CACHE_STATS, to determine which caches to report stats for
6868
private EnumSet<CacheType> includeCaches = EnumSet.noneOf(CacheType.class);
6969
private String[] levels = new String[0];
70+
private boolean optimizeNodeIndicesStatsOnLevel = false;
71+
7072

7173
/**
7274
* @param flags flags to set. If no flags are supplied, default flags will be set.
@@ -100,6 +102,9 @@ public CommonStatsFlags(StreamInput in) throws IOException {
100102
includeCaches = in.readEnumSet(CacheType.class);
101103
levels = in.readStringArray();
102104
}
105+
if (in.getVersion().onOrAfter(Version.V_2_15_0)) {
106+
optimizeNodeIndicesStatsOnLevel = in.readBoolean();
107+
}
103108
}
104109

105110
@Override
@@ -124,6 +129,9 @@ public void writeTo(StreamOutput out) throws IOException {
124129
out.writeEnumSet(includeCaches);
125130
out.writeStringArrayNullable(levels);
126131
}
132+
if (out.getVersion().onOrAfter(Version.V_2_15_0)){
133+
out.writeBoolean(optimizeNodeIndicesStatsOnLevel);
134+
}
127135
}
128136

129137
/**
@@ -262,6 +270,14 @@ public boolean includeSegmentFileSizes() {
262270
return this.includeSegmentFileSizes;
263271
}
264272

273+
public void optimizeNodeIndicesStatsOnLevel(boolean optimizeNodeIndicesStatsOnLevel) {
274+
this.optimizeNodeIndicesStatsOnLevel = optimizeNodeIndicesStatsOnLevel;
275+
}
276+
277+
public boolean optimizeNodeIndicesStatsOnLevel() {
278+
return this.optimizeNodeIndicesStatsOnLevel;
279+
}
280+
265281
public boolean isSet(Flag flag) {
266282
return flags.contains(flag);
267283
}

‎server/src/main/java/org/opensearch/indices/IndicesService.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -622,7 +622,10 @@ public NodeIndicesStats stats(CommonStatsFlags flags) {
622622
break;
623623
}
624624
}
625-
625+
if (flags.optimizeNodeIndicesStatsOnLevel()) {
626+
logger.info("Picked NodeIndicesStats");
627+
return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats, flags.getLevels());
628+
}
626629
return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats);
627630
}
628631

‎server/src/main/java/org/opensearch/indices/NodeIndicesStats.java

+127-15
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@
3232

3333
package org.opensearch.indices;
3434

35+
import org.apache.logging.log4j.LogManager;
36+
import org.apache.logging.log4j.Logger;
37+
import org.opensearch.Version;
3538
import org.opensearch.action.admin.indices.stats.CommonStats;
3639
import org.opensearch.action.admin.indices.stats.IndexShardStats;
3740
import org.opensearch.action.admin.indices.stats.ShardStats;
@@ -63,6 +66,7 @@
6366

6467
import java.io.IOException;
6568
import java.util.ArrayList;
69+
import java.util.Arrays;
6670
import java.util.HashMap;
6771
import java.util.List;
6872
import java.util.Map;
@@ -75,22 +79,21 @@
7579
@PublicApi(since = "1.0.0")
7680
public class NodeIndicesStats implements Writeable, ToXContentFragment {
7781
private CommonStats stats;
82+
private Map<Index, CommonStats> statsByIndex;
7883
private Map<Index, List<IndexShardStats>> statsByShard;
7984

8085
public NodeIndicesStats(StreamInput in) throws IOException {
8186
stats = new CommonStats(in);
87+
if (in.getVersion().onOrAfter(Version.V_2_15_0)) {
88+
// contains statsByIndex
89+
if (in.readBoolean()) {
90+
statsByIndex = new HashMap<>();
91+
readStatsByIndex(in);
92+
}
93+
}
8294
if (in.readBoolean()) {
83-
int entries = in.readVInt();
8495
statsByShard = new HashMap<>();
85-
for (int i = 0; i < entries; i++) {
86-
Index index = new Index(in);
87-
int indexShardListSize = in.readVInt();
88-
List<IndexShardStats> indexShardStats = new ArrayList<>(indexShardListSize);
89-
for (int j = 0; j < indexShardListSize; j++) {
90-
indexShardStats.add(new IndexShardStats(in));
91-
}
92-
statsByShard.put(index, indexShardStats);
93-
}
96+
readStatsByShards(in);
9497
}
9598
}
9699

@@ -112,6 +115,57 @@ public NodeIndicesStats(CommonStats oldStats, Map<Index, List<IndexShardStats>>
112115
}
113116
}
114117

118+
public NodeIndicesStats(
119+
CommonStats oldStats,
120+
Map<Index, List<IndexShardStats>> statsByShard,
121+
SearchRequestStats searchRequestStats,
122+
String[] levels
123+
) {
124+
// make a total common stats from old ones and current ones
125+
this.stats = oldStats;
126+
for (List<IndexShardStats> shardStatsList : statsByShard.values()) {
127+
for (IndexShardStats indexShardStats : shardStatsList) {
128+
for (ShardStats shardStats : indexShardStats.getShards()) {
129+
stats.add(shardStats.getStats());
130+
}
131+
}
132+
}
133+
134+
if (this.stats.search != null) {
135+
this.stats.search.setSearchRequestStats(searchRequestStats);
136+
}
137+
138+
if (levels != null) {
139+
if (Arrays.stream(levels).anyMatch(NodeIndicesStats.levels.indices::equals)) {
140+
this.statsByIndex = createStatsByIndex(statsByShard);
141+
} else if (Arrays.stream(levels).anyMatch(NodeIndicesStats.levels.shards::equals)) {
142+
this.statsByShard = statsByShard;
143+
}
144+
}
145+
}
146+
147+
private void readStatsByIndex(StreamInput in) throws IOException {
148+
int indexEntries = in.readVInt();
149+
for (int i = 0; i < indexEntries; i++) {
150+
Index index = new Index(in);
151+
CommonStats commonStats = new CommonStats(in);
152+
statsByIndex.put(index, commonStats);
153+
}
154+
}
155+
156+
private void readStatsByShards(StreamInput in) throws IOException {
157+
int entries = in.readVInt();
158+
for (int i = 0; i < entries; i++) {
159+
Index index = new Index(in);
160+
int indexShardListSize = in.readVInt();
161+
List<IndexShardStats> indexShardStats = new ArrayList<>(indexShardListSize);
162+
for (int j = 0; j < indexShardListSize; j++) {
163+
indexShardStats.add(new IndexShardStats(in));
164+
}
165+
statsByShard.put(index, indexShardStats);
166+
}
167+
}
168+
115169
@Nullable
116170
public StoreStats getStore() {
117171
return stats.getStore();
@@ -195,7 +249,31 @@ public RecoveryStats getRecoveryStats() {
195249
@Override
196250
public void writeTo(StreamOutput out) throws IOException {
197251
stats.writeTo(out);
252+
253+
if (out.getVersion().onOrAfter(Version.V_2_15_0)) {
254+
out.writeBoolean(statsByIndex != null);
255+
if (statsByIndex != null) {
256+
writeStatsByIndex(out);
257+
}
258+
}
259+
198260
out.writeBoolean(statsByShard != null);
261+
if (statsByShard != null) {
262+
writeStatsByShards(out);
263+
}
264+
}
265+
266+
private void writeStatsByIndex(StreamOutput out) throws IOException {
267+
if (statsByIndex != null) {
268+
out.writeVInt(statsByIndex.size());
269+
for (Map.Entry<Index, CommonStats> entry : statsByIndex.entrySet()) {
270+
entry.getKey().writeTo(out);
271+
entry.getValue().writeTo(out);
272+
}
273+
}
274+
}
275+
276+
private void writeStatsByShards(StreamOutput out) throws IOException {
199277
if (statsByShard != null) {
200278
out.writeVInt(statsByShard.size());
201279
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
@@ -222,16 +300,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
222300
builder.startObject(Fields.INDICES);
223301
stats.toXContent(builder, params);
224302

225-
if ("indices".equals(level)) {
226-
Map<Index, CommonStats> indexStats = createStatsByIndex();
303+
if (levels.indices.equals(level)) {
227304
builder.startObject(Fields.INDICES);
228-
for (Map.Entry<Index, CommonStats> entry : indexStats.entrySet()) {
305+
if (statsByIndex == null && statsByShard!=null) {
306+
statsByIndex = createStatsByIndex(statsByShard);
307+
}
308+
for (Map.Entry<Index, CommonStats> entry : statsByIndex.entrySet()) {
229309
builder.startObject(entry.getKey().getName());
230310
entry.getValue().toXContent(builder, params);
231311
builder.endObject();
232312
}
233313
builder.endObject();
234-
} else if ("shards".equals(level)) {
314+
} else if (levels.shards.equals(level)) {
235315
builder.startObject("shards");
236316
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
237317
builder.startArray(entry.getKey().getName());
@@ -251,7 +331,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
251331
return builder;
252332
}
253333

254-
private Map<Index, CommonStats> createStatsByIndex() {
334+
private Map<Index, CommonStats> createStatsByIndex(Map<Index, List<IndexShardStats>> statsByShard) {
255335
Map<Index, CommonStats> statsMap = new HashMap<>();
256336
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
257337
if (!statsMap.containsKey(entry.getKey())) {
@@ -276,6 +356,14 @@ public List<IndexShardStats> getShardStats(Index index) {
276356
}
277357
}
278358

359+
public CommonStats getIndexStats(Index index) {
360+
if (statsByIndex == null) {
361+
return null;
362+
} else {
363+
return statsByIndex.get(index);
364+
}
365+
}
366+
279367
/**
280368
* Fields used for parsing and toXContent
281369
*
@@ -284,4 +372,28 @@ public List<IndexShardStats> getShardStats(Index index) {
284372
static final class Fields {
285373
static final String INDICES = "indices";
286374
}
375+
376+
/**
377+
* Levels for the NodeIndicesStats
378+
*/
379+
public enum levels {
380+
node("node"),
381+
indices("indices"),
382+
shards("shards");
383+
384+
private final String name;
385+
386+
levels(String name) {
387+
this.name = name;
388+
}
389+
390+
@Override
391+
public String toString() {
392+
return name;
393+
}
394+
395+
public boolean equals(String value) {
396+
return this.name.equals(value);
397+
}
398+
}
287399
}

‎server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java

+1
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
232232
// If no levels are passed in this results in an empty array.
233233
String[] levels = Strings.splitStringByCommaToArray(request.param("level"));
234234
nodesStatsRequest.indices().setLevels(levels);
235+
nodesStatsRequest.indices().optimizeNodeIndicesStatsOnLevel(true);
235236

236237
return channel -> client.admin().cluster().nodesStats(nodesStatsRequest, new NodesResponseRestListener<>(channel));
237238
}

‎server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ public void processResponse(final NodesInfoResponse nodesInfoResponse) {
138138
NodesStatsRequest nodesStatsRequest = new NodesStatsRequest();
139139
nodesStatsRequest.timeout(request.param("timeout"));
140140
nodesStatsRequest.clear()
141-
.indices(true)
141+
.indices(true).useOptimizedNodeIndicesStats(true)
142142
.addMetrics(
143143
NodesStatsRequest.Metric.JVM.metricName(),
144144
NodesStatsRequest.Metric.OS.metricName(),

0 commit comments

Comments
 (0)