Skip to content

Commit 5597285

Browse files
chiacyuchiacyu
committed
Merge from master
2 parents 4bc69ef + 07394a5 commit 5597285

File tree

27 files changed

+312
-155
lines changed

27 files changed

+312
-155
lines changed

‎.github/workflows/intermittent-test-check.yml

+8-3
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ on:
4848
description: Ratis ref (branch, tag or commit SHA)
4949
default: ''
5050
required: false
51+
java-version:
52+
description: Java version to use
53+
default: '21'
54+
required: true
5155
env:
5256
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
5357
TEST_CLASS: ${{ github.event.inputs.test-class}}
@@ -56,7 +60,8 @@ env:
5660
FAIL_FAST: ${{ github.event.inputs.fail-fast }}
5761
RATIS_REPO: ${{ github.event.inputs.ratis-repo }}
5862
RATIS_VERSION: ${{ github.event.inputs.ratis-ref }}
59-
run-name: ${{ github.event_name == 'workflow_dispatch' && format('{0}#{1}[{2}]-{3}x{4}', inputs.test-class, inputs.test-name, inputs.ref, inputs.splits, inputs.iterations) || '' }}
63+
JAVA_VERSION: ${{ github.event.inputs.java-version }}
64+
run-name: ${{ github.event_name == 'workflow_dispatch' && format('{0}#{1}[{2}]-{3}x{4}-java{5}', inputs.test-class, inputs.test-name, inputs.ref, inputs.splits, inputs.iterations, inputs.java-version) || '' }}
6065
jobs:
6166
prepare-job:
6267
runs-on: ubuntu-24.04
@@ -112,7 +117,7 @@ jobs:
112117
uses: actions/setup-java@v4
113118
with:
114119
distribution: 'temurin'
115-
java-version: 8
120+
java-version: ${{ github.event.inputs.java-version }}
116121
- name: Build (most) of Ozone
117122
run: |
118123
args="-DskipRecon -DskipShade -Dmaven.javadoc.skip=true"
@@ -176,7 +181,7 @@ jobs:
176181
uses: actions/setup-java@v4
177182
with:
178183
distribution: 'temurin'
179-
java-version: 8
184+
java-version: ${{ github.event.inputs.java-version }}
180185
- name: Execute tests
181186
run: |
182187
if [[ -e "${{ steps.download-ozone-repo.outputs.download-path }}" ]]; then

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java

-6
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.hadoop.hdds.conf.ConfigurationSource;
2525
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
2626
import org.apache.hadoop.ozone.OzoneConsts;
27-
import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
2827
import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
2928
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
3029
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
@@ -88,11 +87,6 @@ public EndpointStateMachine.EndPointStates call() throws Exception {
8887
// Check HddsVolumes
8988
checkVolumeSet(ozoneContainer.getVolumeSet(), scmId, clusterId);
9089

91-
DatanodeLayoutStorage layoutStorage
92-
= new DatanodeLayoutStorage(configuration);
93-
layoutStorage.setClusterId(clusterId);
94-
layoutStorage.persistCurrentState();
95-
9690
// Start the container services after getting the version information
9791
ozoneContainer.start(clusterId);
9892
}

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java

+6
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import org.apache.hadoop.hdds.utils.db.Table;
6666
import org.apache.hadoop.hdds.utils.db.TableIterator;
6767
import org.apache.hadoop.ozone.HddsDatanodeService;
68+
import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
6869
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
6970
import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService;
7071
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@ -484,6 +485,11 @@ public void start(String clusterId) throws IOException {
484485
return;
485486
}
486487

488+
DatanodeLayoutStorage layoutStorage
489+
= new DatanodeLayoutStorage(config);
490+
layoutStorage.setClusterId(clusterId);
491+
layoutStorage.persistCurrentState();
492+
487493
buildContainerSet();
488494

489495
// Start background volume checks, which will begin after the configured

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java

+11
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.hadoop.ozone.container.common;
1919

20+
import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED;
2021
import static org.mockito.Mockito.any;
2122
import static org.mockito.Mockito.mock;
2223
import static org.mockito.Mockito.when;
@@ -46,6 +47,7 @@
4647
import org.apache.hadoop.ipc.ProtobufRpcEngine;
4748
import org.apache.hadoop.ipc.RPC;
4849
import org.apache.hadoop.net.NetUtils;
50+
import org.apache.hadoop.ozone.HddsDatanodeService;
4951
import org.apache.hadoop.ozone.OzoneConfigKeys;
5052
import org.apache.hadoop.ozone.container.ContainerTestHelper;
5153
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -350,4 +352,13 @@ public static XceiverServerRatis newXceiverServerRatis(
350352
getNoopContainerDispatcher(), getEmptyContainerController(),
351353
null, null);
352354
}
355+
356+
/** Initialize {@link DatanodeLayoutStorage}. Normally this is done during {@link HddsDatanodeService} start,
357+
* have to do the same for tests that create {@link OzoneContainer} manually. */
358+
public static void initializeDatanodeLayout(ConfigurationSource conf, DatanodeDetails dn) throws IOException {
359+
DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(conf, dn.getUuidString());
360+
if (layoutStorage.getState() != INITIALIZED) {
361+
layoutStorage.initialize();
362+
}
363+
}
353364
}

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachine.java

+35-26
Original file line numberDiff line numberDiff line change
@@ -123,12 +123,15 @@ public void testWriteFailure(boolean failWithException) throws ExecutionExceptio
123123
RaftProtos.LogEntryProto entry = mock(RaftProtos.LogEntryProto.class);
124124
when(entry.getTerm()).thenReturn(1L);
125125
when(entry.getIndex()).thenReturn(1L);
126+
RaftProtos.LogEntryProto entryNext = mock(RaftProtos.LogEntryProto.class);
127+
when(entryNext.getTerm()).thenReturn(1L);
128+
when(entryNext.getIndex()).thenReturn(2L);
126129
TransactionContext trx = mock(TransactionContext.class);
127130
ContainerStateMachine.Context context = mock(ContainerStateMachine.Context.class);
128131
when(trx.getStateMachineContext()).thenReturn(context);
129132

130133
setUpMockDispatcherReturn(failWithException);
131-
setUpMockRequestProtoReturn(context, "Test Data", 1, 1);
134+
setUpMockRequestProto(context);
132135

133136
AtomicReference<Throwable> throwable = new AtomicReference<>(null);
134137
Function<Throwable, ? extends Message> throwableSetter = getThrowableSetter(throwable);
@@ -141,9 +144,14 @@ public void testWriteFailure(boolean failWithException) throws ExecutionExceptio
141144
assertResults(failWithException, throwable);
142145

143146
// Writing data to another container(containerId 2) should also fail.
144-
setUpMockRequestProtoReturn(context, "Test Data", 2, 1);
145-
146-
stateMachine.write(entry, trx).exceptionally(throwableSetter).get();
147+
when(context.getRequestProto()).thenReturn(ContainerProtos.ContainerCommandRequestProto.newBuilder()
148+
.setCmdType(ContainerProtos.Type.WriteChunk).setWriteChunk(
149+
ContainerProtos.WriteChunkRequestProto.newBuilder().setData(ByteString.copyFromUtf8("Test Data"))
150+
.setBlockID(
151+
ContainerProtos.DatanodeBlockID.newBuilder().setContainerID(2).setLocalID(1).build()).build())
152+
.setContainerID(2)
153+
.setDatanodeUuid(UUID.randomUUID().toString()).build());
154+
stateMachine.write(entryNext, trx).exceptionally(throwableSetter).get();
147155
verify(dispatcher, times(0)).dispatch(any(ContainerProtos.ContainerCommandRequestProto.class),
148156
any(DispatcherContext.class));
149157
assertInstanceOf(StorageContainerException.class, throwable.get());
@@ -162,33 +170,22 @@ public final void setUpMockDispatcherReturn(boolean failWithException) {
162170
}
163171
}
164172

165-
public final void setUpMockRequestProtoReturn(ContainerStateMachine.Context context, String content,
166-
int containerId, int localId) {
173+
public final void setUpMockRequestProto(ContainerStateMachine.Context context) {
167174
when(context.getRequestProto()).thenReturn(ContainerProtos.ContainerCommandRequestProto.newBuilder()
168175
.setCmdType(ContainerProtos.Type.WriteChunk).setWriteChunk(
169-
ContainerProtos.WriteChunkRequestProto.newBuilder().setData(ByteString.copyFromUtf8(content))
176+
ContainerProtos.WriteChunkRequestProto.newBuilder().setData(ByteString.copyFromUtf8("Test Data"))
170177
.setBlockID(
171-
ContainerProtos.DatanodeBlockID.newBuilder().setContainerID(containerId)
172-
.setLocalID(localId).build()).build())
173-
.setContainerID(containerId)
174-
.setDatanodeUuid(UUID.randomUUID().toString()).build());
175-
}
176-
177-
public final void setUpMockLogProtoReturn(ContainerStateMachine.Context context, int containerId, int localId) {
178-
when(context.getLogProto()).thenReturn(ContainerProtos.ContainerCommandRequestProto.newBuilder()
179-
.setCmdType(ContainerProtos.Type.WriteChunk).setWriteChunk(
180-
ContainerProtos.WriteChunkRequestProto.newBuilder().setBlockID(
181-
ContainerProtos.DatanodeBlockID.newBuilder().setContainerID(containerId)
182-
.setLocalID(localId).build()).build())
183-
.setContainerID(containerId)
178+
ContainerProtos.DatanodeBlockID.newBuilder().setContainerID(1).setLocalID(1).build()).build())
179+
.setContainerID(1)
184180
.setDatanodeUuid(UUID.randomUUID().toString()).build());
185181
}
186182

187183
public final Function<Throwable, ? extends Message> getThrowableSetter(AtomicReference<Throwable> throwable) {
188-
return t -> {
184+
Function<Throwable, ? extends Message> throwableSetter = t -> {
189185
throwable.set(t);
190186
return null;
191187
};
188+
return throwableSetter;
192189
}
193190

194191
public final void assertResults(boolean failWithException, AtomicReference<Throwable> throwable) {
@@ -215,8 +212,12 @@ public void testApplyTransactionFailure(boolean failWithException) throws Execut
215212

216213
setUpMockDispatcherReturn(failWithException);
217214
// Failing apply transaction on congtainer 1.
218-
setUpMockLogProtoReturn(context, 1, 1);
219-
215+
when(context.getLogProto()).thenReturn(ContainerProtos.ContainerCommandRequestProto.newBuilder()
216+
.setCmdType(ContainerProtos.Type.WriteChunk).setWriteChunk(
217+
ContainerProtos.WriteChunkRequestProto.newBuilder().setBlockID(
218+
ContainerProtos.DatanodeBlockID.newBuilder().setContainerID(1).setLocalID(1).build()).build())
219+
.setContainerID(1)
220+
.setDatanodeUuid(UUID.randomUUID().toString()).build());
220221
AtomicReference<Throwable> throwable = new AtomicReference<>(null);
221222
Function<Throwable, ? extends Message> throwableSetter = getThrowableSetter(throwable);
222223
//apply transaction will fail because of runtime exception thrown by dispatcher, which marks the first
@@ -237,7 +238,12 @@ public void testApplyTransactionFailure(boolean failWithException) throws Execut
237238

238239
// Another apply transaction on a different container 2 shouldn't fail because the previous apply transaction
239240
// failure was only on container 1.
240-
setUpMockLogProtoReturn(context, 2, 1);
241+
when(context.getLogProto()).thenReturn(ContainerProtos.ContainerCommandRequestProto.newBuilder()
242+
.setCmdType(ContainerProtos.Type.WriteChunk).setWriteChunk(
243+
ContainerProtos.WriteChunkRequestProto.newBuilder().setBlockID(
244+
ContainerProtos.DatanodeBlockID.newBuilder().setContainerID(2).setLocalID(1).build()).build())
245+
.setContainerID(2)
246+
.setDatanodeUuid(UUID.randomUUID().toString()).build());
241247

242248
reset(dispatcher);
243249
throwable.set(null);
@@ -274,9 +280,12 @@ public void testWriteTimout() throws Exception {
274280
return null;
275281
}).when(dispatcher).dispatch(any(), any());
276282

277-
setUpMockRequestProtoReturn(context, "Test Data", 1, 1);
283+
setUpMockRequestProto(context);
278284
AtomicReference<Throwable> throwable = new AtomicReference<>(null);
279-
Function<Throwable, ? extends Message> throwableSetter = getThrowableSetter(throwable);
285+
Function<Throwable, ? extends Message> throwableSetter = t -> {
286+
throwable.set(t);
287+
return null;
288+
};
280289
Field writeChunkWaitMaxNs = stateMachine.getClass().getDeclaredField("writeChunkWaitMaxNs");
281290
writeChunkWaitMaxNs.setAccessible(true);
282291
writeChunkWaitMaxNs.set(stateMachine, 1000_000_000);

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineFollower.java

-3
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,9 @@
1717

1818
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
1919

20-
import org.apache.ozone.test.tag.Flaky;
21-
2220
/**
2321
* Test class to ContainerStateMachine class for follower.
2422
*/
25-
@Flaky("HDDS-12602")
2623
public class TestContainerStateMachineFollower extends TestContainerStateMachine {
2724
public TestContainerStateMachineFollower() {
2825
super(false);

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestContainerStateMachineLeader.java

-3
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,9 @@
1717

1818
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
1919

20-
import org.apache.ozone.test.tag.Flaky;
21-
2220
/**
2321
* Test class to ContainerStateMachine class for leader.
2422
*/
25-
@Flaky("HDDS-12602")
2623
public class TestContainerStateMachineLeader extends TestContainerStateMachine {
2724
public TestContainerStateMachineLeader() {
2825
super(true);

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/CodecRegistry.java

+2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.HashMap;
2424
import java.util.List;
2525
import java.util.Map;
26+
import java.util.Objects;
2627
import org.apache.commons.lang3.ClassUtils;
2728

2829
/**
@@ -61,6 +62,7 @@ private CodecMap(Map<Class<?>, Codec<?>> map) {
6162
}
6263

6364
<T> Codec<T> get(Class<T> clazz) {
65+
Objects.requireNonNull(clazz, "clazz == null");
6466
final Codec<?> codec = map.get(clazz);
6567
return (Codec<T>) codec;
6668
}

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java

+13
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,19 @@ <KEY, VALUE> Table<KEY, VALUE> getTable(String name,
7373
Class<KEY> keyType, Class<VALUE> valueType,
7474
TableCache.CacheType cacheType) throws IOException;
7575

76+
/**
77+
* Gets table store with implict key/value conversion.
78+
*
79+
* @param name - table name
80+
* @param keyCodec - key codec
81+
* @param valueCodec - value codec
82+
* @param cacheType - cache type
83+
* @return - Table Store
84+
* @throws IOException
85+
*/
86+
<KEY, VALUE> TypedTable<KEY, VALUE> getTable(
87+
String name, Codec<KEY> keyCodec, Codec<VALUE> valueCodec, TableCache.CacheType cacheType) throws IOException;
88+
7689
/**
7790
* Lists the Known list of Tables in a DB.
7891
*

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java

+6
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,12 @@ public <K, V> TypedTable<K, V> getTable(String name,
300300
valueType);
301301
}
302302

303+
@Override
304+
public <K, V> TypedTable<K, V> getTable(
305+
String name, Codec<K> keyCodec, Codec<V> valueCodec, TableCache.CacheType cacheType) throws IOException {
306+
return new TypedTable<>(getTable(name), keyCodec, valueCodec, cacheType);
307+
}
308+
303309
@Override
304310
public <K, V> Table<K, V> getTable(String name,
305311
Class<K> keyType, Class<V> valueType,

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java

+24-14
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.hadoop.hdds.utils.db.cache.PartialTableCache;
4141
import org.apache.hadoop.hdds.utils.db.cache.TableCache;
4242
import org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType;
43+
import org.apache.hadoop.hdds.utils.db.cache.TableNoCache;
4344
import org.apache.ratis.util.Preconditions;
4445
import org.apache.ratis.util.function.CheckedBiFunction;
4546

@@ -88,19 +89,27 @@ public class TypedTable<KEY, VALUE> implements Table<KEY, VALUE> {
8889
*/
8990
TypedTable(RDBTable rawTable, CodecRegistry codecRegistry, Class<KEY> keyType, Class<VALUE> valueType,
9091
CacheType cacheType) throws IOException {
91-
this.rawTable = Objects.requireNonNull(rawTable, "rawTable==null");
92-
Objects.requireNonNull(codecRegistry, "codecRegistry == null");
93-
94-
Objects.requireNonNull(keyType, "keyType == null");
95-
this.keyCodec = codecRegistry.getCodecFromClass(keyType);
96-
Objects.requireNonNull(keyCodec, "keyCodec == null");
92+
this(rawTable, codecRegistry.getCodecFromClass(keyType), codecRegistry.getCodecFromClass(valueType),
93+
cacheType);
94+
}
9795

98-
Objects.requireNonNull(valueType, "valueType == null");
99-
this.valueCodec = codecRegistry.getCodecFromClass(valueType);
100-
Objects.requireNonNull(valueCodec, "valueCodec == null");
96+
/**
97+
* Create an TypedTable from the raw table with specified cache type.
98+
*
99+
* @param rawTable The underlying (untyped) table in RocksDB.
100+
* @param keyCodec The key codec.
101+
* @param valueCodec The value codec.
102+
* @param cacheType How to cache the entries?
103+
* @throws IOException
104+
*/
105+
public TypedTable(
106+
RDBTable rawTable, Codec<KEY> keyCodec, Codec<VALUE> valueCodec, CacheType cacheType) throws IOException {
107+
this.rawTable = Objects.requireNonNull(rawTable, "rawTable==null");
108+
this.keyCodec = Objects.requireNonNull(keyCodec, "keyCodec == null");
109+
this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec == null");
101110

102-
this.info = getClassSimpleName(getClass()) + "-" + getName()
103-
+ "(" + getClassSimpleName(keyType) + "->" + getClassSimpleName(valueType) + ")";
111+
this.info = getClassSimpleName(getClass()) + "-" + getName() + "(" + getClassSimpleName(keyCodec.getTypeClass())
112+
+ "->" + getClassSimpleName(valueCodec.getTypeClass()) + ")";
104113

105114
this.supportCodecBuffer = keyCodec.supportCodecBuffer()
106115
&& valueCodec.supportCodecBuffer();
@@ -109,8 +118,7 @@ public class TypedTable<KEY, VALUE> implements Table<KEY, VALUE> {
109118
if (cacheType == CacheType.FULL_CACHE) {
110119
cache = new FullTableCache<>(threadNamePrefix);
111120
//fill cache
112-
try (TableIterator<KEY, ? extends KeyValue<KEY, VALUE>> tableIterator =
113-
iterator()) {
121+
try (TableIterator<KEY, ? extends KeyValue<KEY, VALUE>> tableIterator = iterator()) {
114122

115123
while (tableIterator.hasNext()) {
116124
KeyValue< KEY, VALUE > kv = tableIterator.next();
@@ -122,8 +130,10 @@ public class TypedTable<KEY, VALUE> implements Table<KEY, VALUE> {
122130
CacheValue.get(EPOCH_DEFAULT, kv.getValue()));
123131
}
124132
}
125-
} else {
133+
} else if (cacheType == CacheType.PARTIAL_CACHE) {
126134
cache = new PartialTableCache<>(threadNamePrefix);
135+
} else {
136+
cache = TableNoCache.instance();
127137
}
128138
}
129139

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/PartialTableCache.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -161,8 +161,7 @@ public CacheResult<VALUE> lookup(CacheKey<KEY> cachekey) {
161161
CacheValue<VALUE> cachevalue = cache.get(cachekey);
162162
statsRecorder.recordValue(cachevalue);
163163
if (cachevalue == null) {
164-
return new CacheResult<>(CacheResult.CacheStatus.MAY_EXIST,
165-
null);
164+
return (CacheResult<VALUE>) MAY_EXIST;
166165
} else {
167166
if (cachevalue.getCacheValue() != null) {
168167
return new CacheResult<>(CacheResult.CacheStatus.EXISTS, cachevalue);

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCache.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
@Private
3535
@Evolving
3636
public interface TableCache<KEY, VALUE> {
37+
CacheResult<?> MAY_EXIST = new CacheResult<>(CacheResult.CacheStatus.MAY_EXIST, null);
3738

3839
/**
3940
* Return the value for the key if it is present, otherwise return null.
@@ -113,7 +114,8 @@ public interface TableCache<KEY, VALUE> {
113114
enum CacheType {
114115
FULL_CACHE, // This mean's the table maintains full cache. Cache and DB
115116
// state are same.
116-
PARTIAL_CACHE // This is partial table cache, cache state is partial state
117+
PARTIAL_CACHE, // This is partial table cache, cache state is partial state
117118
// compared to DB state.
119+
NO_CACHE
118120
}
119121
}

0 commit comments

Comments
 (0)