Skip to content

Adding flattening of strict final fields #1407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: lworld
Choose a base branch
from
Draft

Adding flattening of strict final fields #1407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/hotspot/share/classfile/classFileParser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5490,7 +5490,8 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik,
vk->set_non_atomic_size_in_bytes(_layout_info->_non_atomic_size_in_bytes);
vk->set_non_atomic_alignment(_layout_info->_non_atomic_alignment);
vk->set_atomic_size_in_bytes(_layout_info->_atomic_layout_size_in_bytes);
vk->set_nullable_size_in_bytes(_layout_info->_nullable_layout_size_in_bytes);
vk->set_nullable_atomic_size_in_bytes(_layout_info->_nullable_atomic_layout_size_in_bytes);
vk->set_nullable_non_atomic_size_in_bytes(_layout_info->_nullable_non_atomic_layout_size_in_bytes);
vk->set_null_marker_offset(_layout_info->_null_marker_offset);
vk->set_default_value_offset(_layout_info->_default_value_offset);
vk->set_null_reset_value_offset(_layout_info->_null_reset_value_offset);
Expand Down
3 changes: 2 additions & 1 deletion src/hotspot/share/classfile/classFileParser.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ class FieldLayoutInfo : public ResourceObj {
int _non_atomic_size_in_bytes;
int _non_atomic_alignment;
int _atomic_layout_size_in_bytes;
int _nullable_layout_size_in_bytes;
int _nullable_atomic_layout_size_in_bytes;
int _nullable_non_atomic_layout_size_in_bytes;
int _null_marker_offset;
int _default_value_offset;
int _null_reset_value_offset;
Expand Down
78 changes: 55 additions & 23 deletions src/hotspot/share/classfile/fieldLayoutBuilder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,12 @@
#include "utilities/powerOfTwo.hpp"

static LayoutKind field_layout_selection(FieldInfo field_info, Array<InlineLayoutInfo>* inline_layout_info_array,
bool use_atomic_flat) {
bool can_use_atomic_flat) {

// The can_use_atomic_flat argument indicates if an atomic flat layout can be used for this field.
// This argument will be false if the container is a loosely consistent value class. Using an atomic layout
// in a container that has no atomicity guarantee creates a risk to see this field's value be subject to
// tearing even if the field's class was declared atomic (non loosely consistent).

if (!UseFieldFlattening) {
return LayoutKind::REFERENCE;
Expand Down Expand Up @@ -66,13 +71,18 @@ static LayoutKind field_layout_selection(FieldInfo field_info, Array<InlineLayou
assert(vk->is_implicitly_constructible(), "null-free fields must be implicitly constructible");
if (vk->must_be_atomic() || AlwaysAtomicAccesses) {
if (vk->is_naturally_atomic() && vk->has_non_atomic_layout()) return LayoutKind::NON_ATOMIC_FLAT;
return (vk->has_atomic_layout() && use_atomic_flat) ? LayoutKind::ATOMIC_FLAT : LayoutKind::REFERENCE;
return (vk->has_atomic_layout() && can_use_atomic_flat) ? LayoutKind::ATOMIC_FLAT : LayoutKind::REFERENCE;
} else {
return vk->has_non_atomic_layout() ? LayoutKind::NON_ATOMIC_FLAT : LayoutKind::REFERENCE;
}
} else {
// To preserve the consistency between the null-marker and the field content, the NULLABLE_NON_ATOMIC_FLAT
// can only be used in containers that have atomicity quarantees (can_use_atomic_flat argument set to true)
if (field_info.access_flags().is_strict() && field_info.access_flags().is_final() && can_use_atomic_flat) {
if (vk->has_nullable_non_atomic_layout()) return LayoutKind::NULLABLE_NON_ATOMIC_FLAT;
}
if (UseNullableValueFlattening && vk->has_nullable_atomic_layout()) {
return use_atomic_flat ? LayoutKind::NULLABLE_ATOMIC_FLAT : LayoutKind::REFERENCE;
return can_use_atomic_flat ? LayoutKind::NULLABLE_ATOMIC_FLAT : LayoutKind::REFERENCE;
} else {
return LayoutKind::REFERENCE;
}
Expand All @@ -92,7 +102,11 @@ static void get_size_and_alignment(InlineKlass* vk, LayoutKind kind, int* size,
case LayoutKind::NULLABLE_ATOMIC_FLAT:
*size = vk->nullable_atomic_size_in_bytes();
*alignment = *size;
break;
break;
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
*size = vk->nullable_non_atomic_size_in_bytes();
*alignment = vk->non_atomic_alignment();
break;
default:
ShouldNotReachHere();
}
Expand Down Expand Up @@ -405,7 +419,8 @@ LayoutRawBlock* FieldLayout::insert_field_block(LayoutRawBlock* slot, LayoutRawB
_null_reset_value_offset = block->offset();
}
}
if (block->block_kind() == LayoutRawBlock::FLAT && block->layout_kind() == LayoutKind::NULLABLE_ATOMIC_FLAT) {
if (block->block_kind() == LayoutRawBlock::FLAT
&& (block->layout_kind() == LayoutKind::NULLABLE_ATOMIC_FLAT || block->layout_kind() == LayoutKind::NULLABLE_NON_ATOMIC_FLAT)) {
int nm_offset = block->inline_klass()->null_marker_offset() - block->inline_klass()->payload_offset() + block->offset();
_field_info->adr_at(block->field_index())->set_null_marker_offset(nm_offset);
_inline_layout_info_array->adr_at(block->field_index())->set_null_marker_offset(nm_offset);
Expand Down Expand Up @@ -567,7 +582,8 @@ void FieldLayout::shift_fields(int shift) {
b->set_offset(b->offset() + shift);
if (b->block_kind() == LayoutRawBlock::REGULAR || b->block_kind() == LayoutRawBlock::FLAT) {
_field_info->adr_at(b->field_index())->set_offset(b->offset());
if (b->layout_kind() == LayoutKind::NULLABLE_ATOMIC_FLAT) {
if (b->block_kind() == LayoutRawBlock::FLAT &&
(b->layout_kind() == LayoutKind::NULLABLE_ATOMIC_FLAT || b->layout_kind() == LayoutKind::NULLABLE_NON_ATOMIC_FLAT)) {
int new_nm_offset = _field_info->adr_at(b->field_index())->null_marker_offset() + shift;
_field_info->adr_at(b->field_index())->set_null_marker_offset(new_nm_offset);
_inline_layout_info_array->adr_at(b->field_index())->set_null_marker_offset(new_nm_offset);
Expand Down Expand Up @@ -619,6 +635,8 @@ static const char* layout_kind_to_string(LayoutKind lk) {
return "ATOMIC_FLAT";
case LayoutKind::NULLABLE_ATOMIC_FLAT:
return "NULLABLE_ATOMIC_FLAT";
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
return "NULLABLE_NON_ATOMIC_FLAT";
case LayoutKind::UNKNOWN:
return "UNKNOWN";
default:
Expand Down Expand Up @@ -737,7 +755,8 @@ FieldLayoutBuilder::FieldLayoutBuilder(const Symbol* classname, ClassLoaderData*
_non_atomic_layout_size_in_bytes(-1),
_non_atomic_layout_alignment(-1),
_atomic_layout_size_in_bytes(-1),
_nullable_layout_size_in_bytes(-1),
_nullable_atomic_layout_size_in_bytes(-1),
_nullable_non_atomic_layout_size_in_bytes(-1),
_fields_size_sum(0),
_declared_non_static_fields_count(0),
_has_non_naturally_atomic_fields(false),
Expand Down Expand Up @@ -1128,10 +1147,9 @@ void FieldLayoutBuilder::compute_inline_class_layout() {
}
}

// Next step is the nullable layout: the layout must include a null marker and must also be atomic
if (UseNullableValueFlattening) {
// Next step is the nullable layouts: they must include a null marker
if (UseNullableValueFlattening || UseNullableNonAtomicValueFlattening) {
// Looking if there's an empty slot inside the layout that could be used to store a null marker
// FIXME: could it be possible to re-use the .empty field as a null marker for empty values?
LayoutRawBlock* b = _layout->first_field_block();
assert(b != nullptr, "A concrete value class must have at least one (possible dummy) field");
int null_marker_offset = -1;
Expand Down Expand Up @@ -1160,20 +1178,28 @@ void FieldLayoutBuilder::compute_inline_class_layout() {
null_marker_offset = marker->offset();
}
}

// Now that the null marker is there, the size of the nullable layout must computed (remember, must be atomic too)
assert(null_marker_offset != -1, "Sanity check");
// Now that the null marker is there, the size of the nullable layout must computed
int new_raw_size = _layout->last_block()->offset() - _layout->first_field_block()->offset();
int nullable_size = round_up_power_of_2(new_raw_size);
if (nullable_size <= (int)MAX_ATOMIC_OP_SIZE) {
_nullable_layout_size_in_bytes = nullable_size;
if (UseNullableNonAtomicValueFlattening) {
_nullable_non_atomic_layout_size_in_bytes = new_raw_size;
_null_marker_offset = null_marker_offset;
} else {
_non_atomic_layout_alignment = _payload_alignment;
}
if (UseNullableValueFlattening) {
// For the nullable atomic layout, the size mut be compatible with the platform capabilities
int nullable_atomic_size = round_up_power_of_2(new_raw_size);
if (nullable_atomic_size <= (int)MAX_ATOMIC_OP_SIZE) {
_nullable_atomic_layout_size_in_bytes = nullable_atomic_size;
_null_marker_offset = null_marker_offset;
}
}
if (_null_marker_offset == -1) { // No nullable layout has been accepted
// If the nullable layout is rejected, the NULL_MARKER block should be removed
// from the layout, otherwise it will appear anyway if the layout is printer
if (!_is_empty_inline_class) { // empty values don't have a dedicated NULL_MARKER block
_layout->remove_null_marker();
}
_null_marker_offset = -1;
}
}
// If the inline class has an atomic or nullable (which is also atomic) layout,
Expand Down Expand Up @@ -1203,11 +1229,11 @@ void FieldLayoutBuilder::compute_inline_class_layout() {
_payload_alignment = required_alignment;
} else {
_atomic_layout_size_in_bytes = -1;
if (has_nullable_atomic_layout() && !_is_empty_inline_class) { // empty values don't have a dedicated NULL_MARKER block
if (has_nullable_atomic_layout() && !has_nullable_non_atomic_layout() && !_is_empty_inline_class) { // empty values don't have a dedicated NULL_MARKER block
_layout->remove_null_marker();
_null_marker_offset = -1;
}
_nullable_layout_size_in_bytes = -1;
_null_marker_offset = -1;
_nullable_atomic_layout_size_in_bytes = -1;
}
} else {
_payload_alignment = required_alignment;
Expand Down Expand Up @@ -1308,7 +1334,8 @@ void FieldLayoutBuilder::epilogue() {
_info->_non_atomic_size_in_bytes = _non_atomic_layout_size_in_bytes;
_info->_non_atomic_alignment = _non_atomic_layout_alignment;
_info->_atomic_layout_size_in_bytes = _atomic_layout_size_in_bytes;
_info->_nullable_layout_size_in_bytes = _nullable_layout_size_in_bytes;
_info->_nullable_atomic_layout_size_in_bytes = _nullable_atomic_layout_size_in_bytes;
_info->_nullable_non_atomic_layout_size_in_bytes = _nullable_non_atomic_layout_size_in_bytes;
_info->_null_marker_offset = _null_marker_offset;
_info->_default_value_offset = _static_layout->default_value_offset();
_info->_null_reset_value_offset = _static_layout->null_reset_value_offset();
Expand Down Expand Up @@ -1381,9 +1408,14 @@ void FieldLayoutBuilder::epilogue() {
st.print_cr("Atomic flat layout: -/-");
}
if (has_nullable_atomic_layout()) {
st.print_cr("Nullable flat layout: %d/%d", _nullable_layout_size_in_bytes, _nullable_layout_size_in_bytes);
st.print_cr("Nullable atomic flat layout: %d/%d", _nullable_atomic_layout_size_in_bytes, _nullable_atomic_layout_size_in_bytes);
} else {
st.print_cr("Nullable atomic flat layout: -/-");
}
if (has_nullable_non_atomic_layout()) {
st.print_cr("Nullable non-atomic flat layout: %d/%d", _nullable_non_atomic_layout_size_in_bytes, _payload_alignment);
} else {
st.print_cr("Nullable flat layout: -/-");
st.print_cr("Nullable non-atomic flat layout: -/-");
}
if (_null_marker_offset != -1) {
st.print_cr("Null marker offset = %d", _null_marker_offset);
Expand Down
11 changes: 7 additions & 4 deletions src/hotspot/share/classfile/fieldLayoutBuilder.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,8 @@ class FieldLayoutBuilder : public ResourceObj {
int _non_atomic_layout_size_in_bytes;
int _non_atomic_layout_alignment;
int _atomic_layout_size_in_bytes;
int _nullable_layout_size_in_bytes;
int _nullable_atomic_layout_size_in_bytes;
int _nullable_non_atomic_layout_size_in_bytes;
int _fields_size_sum;
int _declared_non_static_fields_count;
bool _has_non_naturally_atomic_fields;
Expand All @@ -318,16 +319,18 @@ class FieldLayoutBuilder : public ResourceObj {
GrowableArray<FieldInfo>* field_info, bool is_contended, bool is_inline_type, bool is_abstract_value,
bool must_be_atomic, FieldLayoutInfo* info, Array<InlineLayoutInfo>* inline_layout_info_array);

int payload_offset() const { assert(_payload_offset != -1, "Uninitialized"); return _payload_offset; }
int payload_offset() const { assert(_payload_offset != -1, "Uninitialized"); return _payload_offset; }
int payload_layout_size_in_bytes() const { return _payload_size_in_bytes; }
int payload_layout_alignment() const { assert(_payload_alignment != -1, "Uninitialized"); return _payload_alignment; }
bool has_non_atomic_flat_layout() const { return _non_atomic_layout_size_in_bytes != -1; }
int non_atomic_layout_size_in_bytes() const { return _non_atomic_layout_size_in_bytes; }
int non_atomic_layout_alignment() const { return _non_atomic_layout_alignment; }
bool has_atomic_layout() const { return _atomic_layout_size_in_bytes != -1; }
int atomic_layout_size_in_bytes() const { return _atomic_layout_size_in_bytes; }
bool has_nullable_atomic_layout() const { return _nullable_layout_size_in_bytes != -1; }
int nullable_layout_size_in_bytes() const { return _nullable_layout_size_in_bytes; }
bool has_nullable_atomic_layout() const { return _nullable_atomic_layout_size_in_bytes != -1; }
int nullable_layout_size_in_bytes() const { return _nullable_atomic_layout_size_in_bytes; }
bool has_nullable_non_atomic_layout() const { return _nullable_non_atomic_layout_size_in_bytes != -1; }
int nullable_non_atomic_layout_size_in_bytes() const { return _nullable_non_atomic_layout_size_in_bytes; }
int null_marker_offset() const { return _null_marker_offset; }
bool is_empty_inline_class() const { return _is_empty_inline_class; }

Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/share/oops/flatArrayKlass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@

FlatArrayKlass::FlatArrayKlass(Klass* element_klass, Symbol* name, LayoutKind lk) : ArrayKlass(name, Kind, markWord::flat_array_prototype(lk)) {
assert(element_klass->is_inline_klass(), "Expected Inline");
assert(lk != LayoutKind::NULLABLE_NON_ATOMIC_FLAT, "Layout not supported by arrays yet (needs frozen arrays)");
assert(lk == LayoutKind::NON_ATOMIC_FLAT || lk == LayoutKind::ATOMIC_FLAT || lk == LayoutKind::NULLABLE_ATOMIC_FLAT, "Must be a flat layout");

set_element_klass(InlineKlass::cast(element_klass));
Expand All @@ -81,6 +82,8 @@ FlatArrayKlass::FlatArrayKlass(Klass* element_klass, Symbol* name, LayoutKind lk
assert(!layout_helper_is_null_free(layout_helper()), "Must be");
assert(!prototype_header().is_null_free_array(), "Must be");
break;
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
ShouldNotReachHere();
default:
ShouldNotReachHere();
break;
Expand Down
28 changes: 23 additions & 5 deletions src/hotspot/share/oops/inlineKlass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ void InlineKlass::init_fixed_block() {
set_non_atomic_size_in_bytes(-1);
set_non_atomic_alignment(-1);
set_atomic_size_in_bytes(-1);
set_nullable_size_in_bytes(-1);
set_nullable_atomic_size_in_bytes(-1);
set_null_marker_offset(-1);
}

Expand Down Expand Up @@ -144,6 +144,10 @@ int InlineKlass::layout_size_in_bytes(LayoutKind kind) const {
assert(has_nullable_atomic_layout(), "Layout not available");
return nullable_atomic_size_in_bytes();
break;
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
assert(has_nullable_non_atomic_layout(), "Layout not available");
return nullable_non_atomic_size_in_bytes();
break;
case LayoutKind::BUFFERED:
return payload_size_in_bytes();
break;
Expand All @@ -166,6 +170,10 @@ int InlineKlass::layout_alignment(LayoutKind kind) const {
assert(has_nullable_atomic_layout(), "Layout not available");
return nullable_atomic_size_in_bytes();
break;
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
assert(has_nullable_non_atomic_layout(), "Layout not available");
return non_atomic_alignment();
break;
case LayoutKind::BUFFERED:
return payload_alignment();
break;
Expand All @@ -185,6 +193,9 @@ bool InlineKlass::is_layout_supported(LayoutKind lk) {
case LayoutKind::NULLABLE_ATOMIC_FLAT:
return has_nullable_atomic_layout();
break;
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
return has_nullable_non_atomic_layout();
break;
case LayoutKind::BUFFERED:
return true;
break;
Expand All @@ -197,8 +208,9 @@ void InlineKlass::copy_payload_to_addr(void* src, void* dst, LayoutKind lk, bool
assert(is_layout_supported(lk), "Unsupported layout");
assert(lk != LayoutKind::REFERENCE && lk != LayoutKind::UNKNOWN, "Sanity check");
switch(lk) {
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
case LayoutKind::NULLABLE_ATOMIC_FLAT: {
if (is_payload_marked_as_null((address)src)) {
if (is_payload_marked_as_null((address)src)) {
if (!contains_oops()) {
mark_payload_as_null((address)dst);
return;
Expand Down Expand Up @@ -240,6 +252,7 @@ oop InlineKlass::read_payload_from_addr(oop src, int offset, LayoutKind lk, TRAP
assert(src != nullptr, "Must be");
assert(is_layout_supported(lk), "Unsupported layout");
switch(lk) {
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
case LayoutKind::NULLABLE_ATOMIC_FLAT: {
if (is_payload_marked_as_null((address)((char*)(oopDesc*)src + offset))) {
return nullptr;
Expand All @@ -254,7 +267,7 @@ oop InlineKlass::read_payload_from_addr(oop src, int offset, LayoutKind lk, TRAP
Handle obj_h(THREAD, src);
oop res = allocate_instance_buffer(CHECK_NULL);
copy_payload_to_addr((void*)((char*)(oopDesc*)obj_h() + offset), payload_addr(res), lk, false);
if (lk == LayoutKind::NULLABLE_ATOMIC_FLAT) {
if (lk == LayoutKind::NULLABLE_ATOMIC_FLAT || lk == LayoutKind::NULLABLE_NON_ATOMIC_FLAT) { // Should not happen for NULLABLE_NON_ATOMIC_FLAT but let's play it safe
if(is_payload_marked_as_null(payload_addr(res))) {
return nullptr;
}
Expand All @@ -270,7 +283,7 @@ oop InlineKlass::read_payload_from_addr(oop src, int offset, LayoutKind lk, TRAP
void InlineKlass::write_value_to_addr(oop src, void* dst, LayoutKind lk, bool dest_is_initialized, TRAPS) {
void* src_addr = nullptr;
if (src == nullptr) {
if (lk != LayoutKind::NULLABLE_ATOMIC_FLAT) {
if (lk != LayoutKind::NULLABLE_ATOMIC_FLAT && lk != LayoutKind::NULLABLE_NON_ATOMIC_FLAT) {
THROW_MSG(vmSymbols::java_lang_NullPointerException(), "Value is null");
}
// Writing null to a nullable flat field/element is usually done by writing
Expand All @@ -287,7 +300,7 @@ void InlineKlass::write_value_to_addr(oop src, void* dst, LayoutKind lk, bool de
src_addr = payload_addr(null_reset_value());
} else {
src_addr = payload_addr(src);
if (lk == LayoutKind::NULLABLE_ATOMIC_FLAT) {
if (lk == LayoutKind::NULLABLE_ATOMIC_FLAT || lk == LayoutKind::NULLABLE_NON_ATOMIC_FLAT) {
mark_payload_as_non_null((address)src_addr);
}
}
Expand Down Expand Up @@ -352,6 +365,8 @@ FlatArrayKlass* InlineKlass::flat_array_klass(LayoutKind lk, TRAPS) {
assert(has_nullable_atomic_layout(), "Must be");
adr_flat_array_klass = adr_nullable_atomic_flat_array_klass();
break;
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
ShouldNotReachHere();
default:
ShouldNotReachHere();
}
Expand Down Expand Up @@ -383,6 +398,8 @@ FlatArrayKlass* InlineKlass::flat_array_klass_or_null(LayoutKind lk) {
assert(has_nullable_atomic_layout(), "Must be");
adr_flat_array_klass = adr_nullable_atomic_flat_array_klass();
break;
case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
ShouldNotReachHere();
default:
ShouldNotReachHere();
}
Expand Down Expand Up @@ -447,6 +464,7 @@ int InlineKlass::collect_fields(GrowableArray<SigEntry>* sig, float& max_offset,
int offset = base_off + size_helper()*HeapWordSize - (base_off > 0 ? payload_offset() : 0);
// Null markers are no real fields, add them manually at the end (C2 relies on this) of the flat fields
if (null_marker_offset != -1) {
assert(null_marker_offset != 0, "Must be");
max_offset += 0.1f; // We add the markers "in-between" because they are no real fields
SigEntry::add_entry(sig, T_BOOLEAN, name(), null_marker_offset, max_offset);
count++;
Expand Down
Loading