Skip to content

Commit e42599a

Browse files
smiklosovicsmiklosovic
committed
Expose uncaught exceptions in system_views.uncaught_exceptions table
patch by Stefan Miklosovic; reviewed by Dmitry Konstantinov for CASSANDRA-20858
1 parent 70bcaec commit e42599a

File tree

10 files changed

+674
-16
lines changed

10 files changed

+674
-16
lines changed

‎CHANGES.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
5.1
2+
* Expose uncaught exceptions in system_views.uncaught_exceptions table (CASSANDRA-20858)
23
* Improved observability in AutoRepair to report both expected vs. actual repair bytes and expected vs. actual keyspaces (CASSANDRA-20581)
34
* Execution of CreateTriggerStatement should not rely on external state (CASSANDRA-20287)
45
* Support LIKE expressions in filtering queries (CASSANDRA-17198)

‎src/java/org/apache/cassandra/db/virtual/ExceptionsTable.java‎

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.cassandra.db.virtual;
20+
21+
import java.util.ArrayList;
22+
import java.util.Collections;
23+
import java.util.Date;
24+
import java.util.LinkedHashMap;
25+
import java.util.List;
26+
import java.util.Map;
27+
28+
import com.google.common.annotations.VisibleForTesting;
29+
30+
import org.apache.cassandra.db.DecoratedKey;
31+
import org.apache.cassandra.db.marshal.Int32Type;
32+
import org.apache.cassandra.db.marshal.ListType;
33+
import org.apache.cassandra.db.marshal.TimestampType;
34+
import org.apache.cassandra.db.marshal.UTF8Type;
35+
import org.apache.cassandra.dht.LocalPartitioner;
36+
import org.apache.cassandra.schema.TableMetadata;
37+
import org.apache.cassandra.utils.Clock;
38+
import org.apache.cassandra.utils.logging.AbstractVirtualTableAppender;
39+
40+
public class ExceptionsTable extends AbstractMutableVirtualTable
41+
{
42+
public static final String EXCEPTIONS_TABLE_NAME = "uncaught_exceptions";
43+
public static final String EXCEPTION_CLASS_COLUMN_NAME = "exception_class";
44+
public static final String EXCEPTION_LOCATION_COLUMN_NAME = "exception_location";
45+
public static final String COUNT_COLUMN_NAME = "count";
46+
public static final String LAST_MESSAGE_COLUMN_NAME = "last_message";
47+
public static final String LAST_STACKTRACE_COLUMN_NAME = "last_stacktrace";
48+
public static final String LAST_OCCURRENCE_COLUMN_NAME = "last_occurrence";
49+
50+
/**
51+
* Buffer of uncaught exceptions which happened while virtual table was not initialized.
52+
*/
53+
static final List<ExceptionRow> preInitialisationBuffer = Collections.synchronizedList(new ArrayList<>());
54+
55+
@VisibleForTesting
56+
static volatile ExceptionsTable INSTANCE;
57+
58+
// please be sure operations on this structure are thread-safe
59+
@VisibleForTesting
60+
final BoundedMap buffer;
61+
62+
ExceptionsTable(String keyspace)
63+
{
64+
// for starters capped to 1k, I do not think we need to make this configurable (yet).
65+
this(keyspace, 1000);
66+
}
67+
68+
ExceptionsTable(String keyspace, int maxSize)
69+
{
70+
super(TableMetadata.builder(keyspace, EXCEPTIONS_TABLE_NAME)
71+
.comment("View into uncaught exceptions")
72+
.kind(TableMetadata.Kind.VIRTUAL)
73+
.partitioner(new LocalPartitioner(UTF8Type.instance))
74+
.addPartitionKeyColumn(EXCEPTION_CLASS_COLUMN_NAME, UTF8Type.instance)
75+
.addClusteringColumn(EXCEPTION_LOCATION_COLUMN_NAME, UTF8Type.instance)
76+
.addRegularColumn(COUNT_COLUMN_NAME, Int32Type.instance)
77+
.addRegularColumn(LAST_MESSAGE_COLUMN_NAME, UTF8Type.instance)
78+
.addRegularColumn(LAST_STACKTRACE_COLUMN_NAME, ListType.getInstance(UTF8Type.instance, false))
79+
.addRegularColumn(LAST_OCCURRENCE_COLUMN_NAME, TimestampType.instance)
80+
.build());
81+
82+
this.buffer = new BoundedMap(maxSize);
83+
}
84+
85+
public void flush()
86+
{
87+
for (ExceptionRow row : preInitialisationBuffer)
88+
add(row.exceptionClass, row.exceptionLocation, row.message, row.stackTrace, row.occurrence.getTime());
89+
90+
preInitialisationBuffer.clear();
91+
}
92+
93+
@Override
94+
public DataSet data()
95+
{
96+
SimpleDataSet result = new SimpleDataSet(metadata());
97+
98+
synchronized (buffer)
99+
{
100+
for (Map.Entry<String, LinkedHashMap<String, ExceptionRow>> partition : buffer.entrySet())
101+
{
102+
for (Map.Entry<String, ExceptionRow> entry : partition.getValue().entrySet())
103+
populateRow(result, partition.getKey(), entry.getKey(), entry.getValue());
104+
}
105+
}
106+
107+
return result;
108+
}
109+
110+
@Override
111+
public DataSet data(DecoratedKey partitionKey)
112+
{
113+
SimpleDataSet result = new SimpleDataSet(metadata());
114+
115+
synchronized (buffer)
116+
{
117+
String exceptionClass = UTF8Type.instance.getSerializer().deserialize(partitionKey.getKey());
118+
LinkedHashMap<String, ExceptionRow> partition = buffer.get(exceptionClass);
119+
120+
if (partition != null)
121+
{
122+
for (Map.Entry<String, ExceptionRow> row : partition.entrySet())
123+
populateRow(result, exceptionClass, row.getKey(), row.getValue());
124+
}
125+
}
126+
127+
return result;
128+
}
129+
130+
private void populateRow(SimpleDataSet result, String exceptionClass, String exceptionLocation, ExceptionRow row)
131+
{
132+
result.row(exceptionClass, exceptionLocation)
133+
.column(COUNT_COLUMN_NAME, row.count)
134+
.column(LAST_MESSAGE_COLUMN_NAME, row.message)
135+
.column(LAST_STACKTRACE_COLUMN_NAME, row.stackTrace)
136+
.column(LAST_OCCURRENCE_COLUMN_NAME, row.occurrence);
137+
}
138+
139+
@Override
140+
public void truncate()
141+
{
142+
synchronized (buffer)
143+
{
144+
buffer.clear();
145+
}
146+
}
147+
148+
static List<String> extractStacktrace(StackTraceElement[] stackTraceArray)
149+
{
150+
List<String> result = new ArrayList<>(stackTraceArray.length);
151+
152+
for (StackTraceElement element : stackTraceArray)
153+
result.add(element.toString());
154+
155+
return result;
156+
}
157+
158+
public static void persist(Throwable t)
159+
{
160+
if (INSTANCE == null)
161+
INSTANCE = AbstractVirtualTableAppender.getVirtualTable(ExceptionsTable.class, EXCEPTIONS_TABLE_NAME);
162+
163+
Throwable toPersist = t;
164+
165+
while (toPersist.getCause() != null)
166+
toPersist = toPersist.getCause();
167+
168+
List<String> stackTrace = extractStacktrace(toPersist.getStackTrace());
169+
long now = Clock.Global.currentTimeMillis();
170+
171+
if (INSTANCE != null)
172+
{
173+
INSTANCE.add(toPersist.getClass().getName(),
174+
stackTrace.get(0),
175+
toPersist.getMessage(),
176+
stackTrace,
177+
now);
178+
}
179+
else
180+
{
181+
preInitialisationBuffer.add(new ExceptionRow(toPersist.getClass().getName(),
182+
stackTrace.get(0),
183+
0,
184+
toPersist.getMessage(),
185+
stackTrace,
186+
now));
187+
}
188+
}
189+
190+
/**
191+
* Adds entry to internal buffer.
192+
*
193+
* @param exceptionClass exception class of uncaught exception
194+
* @param exceptionLocation location where that exception was thrown
195+
* @param message message of given exception
196+
* @param stackTrace whole stacktrace of given exception
197+
* @param occurrenceTime time when given exception ocurred
198+
*/
199+
private void add(String exceptionClass,
200+
String exceptionLocation,
201+
String message,
202+
List<String> stackTrace,
203+
long occurrenceTime)
204+
{
205+
synchronized (buffer)
206+
{
207+
Map<String, ExceptionRow> exceptionRowWithLocation = buffer.computeIfAbsent(exceptionClass, (classToAdd) -> new LinkedHashMap<>());
208+
ExceptionRow exceptionRow = exceptionRowWithLocation.get(exceptionLocation);
209+
if (exceptionRow == null)
210+
{
211+
// exception class and location can be null for value as we have it as part of keys already
212+
exceptionRow = new ExceptionRow(null, null, 1, message, stackTrace, occurrenceTime);
213+
exceptionRowWithLocation.put(exceptionLocation, exceptionRow);
214+
// not important, can be null
215+
// we need to do this, because if we add into a map which is
216+
// a value of some buffer key, we might exceed the number
217+
// of overall entries in the buffer
218+
buffer.removeEldestEntry(null);
219+
}
220+
else
221+
{
222+
exceptionRow.count += 1;
223+
exceptionRow.message = message;
224+
exceptionRow.stackTrace = stackTrace;
225+
exceptionRow.occurrence = new Date(occurrenceTime);
226+
}
227+
}
228+
}
229+
230+
static final class ExceptionRow
231+
{
232+
final String exceptionClass;
233+
final String exceptionLocation;
234+
int count;
235+
String message;
236+
List<String> stackTrace;
237+
Date occurrence;
238+
239+
/**
240+
* @param exceptionClass exception class of uncaught exception
241+
* @param exceptionLocation location where that exception was thrown
242+
* @param message message of given exception
243+
* @param stackTrace whole stacktrace of given exception
244+
* @param occurrenceTime time when given exception ocurred, in milliseconds from epoch
245+
*/
246+
ExceptionRow(String exceptionClass,
247+
String exceptionLocation,
248+
int count,
249+
String message,
250+
List<String> stackTrace,
251+
long occurrenceTime)
252+
{
253+
this.exceptionClass = exceptionClass;
254+
this.exceptionLocation = exceptionLocation;
255+
this.count = count;
256+
this.stackTrace = stackTrace;
257+
this.message = message;
258+
this.occurrence = new Date(occurrenceTime);
259+
}
260+
}
261+
262+
@VisibleForTesting
263+
static class BoundedMap extends LinkedHashMap<String, LinkedHashMap<String, ExceptionRow>>
264+
{
265+
private final int maxSize;
266+
267+
public BoundedMap(int maxSize)
268+
{
269+
if (maxSize <= 0)
270+
throw new IllegalArgumentException("maxSize has to be bigger than 0");
271+
272+
this.maxSize = maxSize;
273+
}
274+
275+
@Override
276+
protected boolean removeEldestEntry(Map.Entry<String, LinkedHashMap<String, ExceptionRow>> eldest)
277+
{
278+
if (computeSize() > maxSize)
279+
{
280+
String oldestExceptionClass = null;
281+
String oldestExceptionLocation = null;
282+
long oldestLastOccurrence = Long.MAX_VALUE;
283+
for (Map.Entry<String, LinkedHashMap<String, ExceptionRow>> entry : entrySet())
284+
{
285+
for (Map.Entry<String, ExceptionRow> entryInEntry : entry.getValue().entrySet())
286+
{
287+
long currentLastOccurrence = entryInEntry.getValue().occurrence.getTime();
288+
if (currentLastOccurrence < oldestLastOccurrence)
289+
{
290+
oldestExceptionLocation = entryInEntry.getKey();
291+
oldestExceptionClass = entry.getKey();
292+
oldestLastOccurrence = currentLastOccurrence;
293+
}
294+
}
295+
}
296+
297+
if (oldestLastOccurrence < Long.MAX_VALUE)
298+
{
299+
LinkedHashMap<String, ExceptionRow> aMap = get(oldestExceptionClass);
300+
if (aMap.size() == 1)
301+
remove(oldestExceptionClass);
302+
else
303+
aMap.remove(oldestExceptionLocation);
304+
}
305+
}
306+
307+
// always returning false as per method's contract saying that
308+
// overrides might modify the map directly but in that case it must return false
309+
return false;
310+
}
311+
312+
private int computeSize()
313+
{
314+
int size = 0;
315+
316+
for (LinkedHashMap<String, ExceptionRow> value : values())
317+
size += value.size();
318+
319+
return size;
320+
}
321+
}
322+
}

‎src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ private SystemViewsKeyspace()
7171
.addAll(CIDRFilteringMetricsTable.getAll(VIRTUAL_VIEWS))
7272
.addAll(StorageAttachedIndexTables.getAll(VIRTUAL_VIEWS))
7373
.addAll(AccordVirtualTables.getAll(VIRTUAL_VIEWS))
74+
.add(new ExceptionsTable(VIRTUAL_VIEWS))
7475
.build());
7576
}
7677
}

‎src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,16 @@ public void register(VirtualKeyspace keyspace)
4747
keyspace.tables().forEach(t -> virtualTables.put(t.metadata().id, t));
4848
}
4949

50+
public void unregister(VirtualKeyspace keyspace)
51+
{
52+
VirtualKeyspace virtualKeyspace = virtualKeyspaces.get(keyspace.name());
53+
if (virtualKeyspace == null)
54+
return;
55+
56+
keyspace.tables().forEach(t -> virtualTables.remove(t.metadata().id));
57+
virtualKeyspaces.remove(keyspace.name());
58+
}
59+
5060
@Nullable
5161
public VirtualKeyspace getKeyspaceNullable(String name)
5262
{

‎src/java/org/apache/cassandra/service/CassandraDaemon.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.Collection;
2929
import java.util.HashSet;
3030
import java.util.List;
31+
import java.util.Optional;
3132
import java.util.Set;
3233
import java.util.concurrent.ExecutionException;
3334
import java.util.concurrent.TimeUnit;
@@ -59,6 +60,7 @@
5960
import org.apache.cassandra.db.SystemKeyspaceMigrator41;
6061
import org.apache.cassandra.db.commitlog.CommitLog;
6162
import org.apache.cassandra.db.virtual.AccordDebugKeyspace;
63+
import org.apache.cassandra.db.virtual.ExceptionsTable;
6264
import org.apache.cassandra.db.virtual.LogMessagesTable;
6365
import org.apache.cassandra.db.virtual.SlowQueriesTable;
6466
import org.apache.cassandra.db.virtual.SystemViewsKeyspace;
@@ -95,6 +97,7 @@
9597
import org.apache.cassandra.utils.NativeLibrary;
9698
import org.apache.cassandra.utils.concurrent.Future;
9799
import org.apache.cassandra.utils.concurrent.FutureCombiner;
100+
import org.apache.cassandra.utils.logging.AbstractVirtualTableAppender;
98101
import org.apache.cassandra.utils.logging.LoggingSupportFactory;
99102
import org.apache.cassandra.utils.logging.SlowQueriesAppender;
100103
import org.apache.cassandra.utils.logging.VirtualTableAppender;
@@ -569,6 +572,10 @@ public void setupVirtualKeyspaces()
569572
LoggingSupportFactory.getLoggingSupport()
570573
.getAppender(SlowQueriesAppender.class, SlowQueriesAppender.APPENDER_NAME)
571574
.ifPresent(appender -> appender.flushBuffer(SlowQueriesTable.class, SlowQueriesTable.TABLE_NAME));
575+
576+
// populate exceptions table with entries while they were thrown but virtual tables were not registered yet
577+
Optional.ofNullable(AbstractVirtualTableAppender.getVirtualTable(ExceptionsTable.class, ExceptionsTable.EXCEPTIONS_TABLE_NAME))
578+
.ifPresent(ExceptionsTable::flush);
572579
}
573580

574581
public synchronized void initializeClientTransports()

0 commit comments

Comments
 (0)