Skip to content

Commit 41d1b78

Browse files
authored
Support combine multiple add vertice and edges operation. (alibaba#241)
* Support combine multiple add vertice and edges operation.
1 parent d858a36 commit 41d1b78

File tree

19 files changed

+180
-216
lines changed

19 files changed

+180
-216
lines changed

.github/workflows/ci.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030
run:
3131
shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
3232
container:
33-
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.14
33+
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.15
3434
options:
3535
--shm-size 4096m
3636
strategy:
@@ -135,7 +135,7 @@ jobs:
135135
defaults:
136136
run:
137137
shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
138-
container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.14
138+
container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.15
139139
steps:
140140
- name: Install Dependencies
141141
run: |
@@ -175,7 +175,7 @@ jobs:
175175
defaults:
176176
run:
177177
shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
178-
container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.14
178+
container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.15
179179
steps:
180180
- name: Install Dependencies
181181
run: |

.github/workflows/local.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ jobs:
7878
7979
- name: Install libvineyard
8080
run: |
81-
git clone -b v0.1.14 --single-branch --depth=1 https://github.com/alibaba/libvineyard.git
81+
git clone -b v0.1.15 --single-branch --depth=1 https://github.com/alibaba/libvineyard.git
8282
cd libvineyard
8383
git submodule update --init
8484
mkdir build && pushd build

analytical_engine/CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ endif ()
109109
find_package(libgrapelite REQUIRED)
110110
include_directories(${LIBGRAPELITE_INCLUDE_DIRS})
111111

112-
find_package(vineyard 0.1.14 REQUIRED)
112+
find_package(vineyard 0.1.15 REQUIRED)
113113
include_directories(${VINEYARD_INCLUDE_DIRS})
114114
add_compile_options(-DENABLE_SELECTOR)
115115

analytical_engine/core/grape_instance.cc

+4-33
Original file line numberDiff line numberDiff line change
@@ -613,7 +613,7 @@ bl::result<rpc::GraphDef> GrapeInstance::copyGraph(
613613
return dst_wrapper->graph_def();
614614
}
615615

616-
bl::result<rpc::GraphDef> GrapeInstance::addVertices(
616+
bl::result<rpc::GraphDef> GrapeInstance::addLabelsToGraph(
617617
const rpc::GSParams& params) {
618618
BOOST_LEAF_AUTO(graph_name, params.Get<std::string>(rpc::GRAPH_NAME));
619619
BOOST_LEAF_AUTO(
@@ -630,31 +630,7 @@ bl::result<rpc::GraphDef> GrapeInstance::addVertices(
630630
BOOST_LEAF_AUTO(graph_utils,
631631
object_manager_.GetObject<PropertyGraphUtils>(type_sig));
632632
std::string dst_graph_name = "graph_" + generateId();
633-
BOOST_LEAF_AUTO(dst_wrapper, graph_utils->AddVerticesToGraph(
634-
src_frag_id, comm_spec_, *client_,
635-
dst_graph_name, params));
636-
BOOST_LEAF_CHECK(object_manager_.PutObject(dst_wrapper));
637-
638-
return dst_wrapper->graph_def();
639-
}
640-
641-
bl::result<rpc::GraphDef> GrapeInstance::addEdges(const rpc::GSParams& params) {
642-
BOOST_LEAF_AUTO(graph_name, params.Get<std::string>(rpc::GRAPH_NAME));
643-
BOOST_LEAF_AUTO(
644-
src_wrapper,
645-
object_manager_.GetObject<ILabeledFragmentWrapper>(graph_name));
646-
if (src_wrapper->graph_def().graph_type() != rpc::ARROW_PROPERTY) {
647-
RETURN_GS_ERROR(vineyard::ErrorCode::kInvalidOperationError,
648-
"AddEdges is only avaiable for ArrowFragment");
649-
}
650-
651-
auto src_frag_id =
652-
std::static_pointer_cast<vineyard::Object>(src_wrapper->fragment())->id();
653-
BOOST_LEAF_AUTO(type_sig, params.Get<std::string>(rpc::TYPE_SIGNATURE));
654-
BOOST_LEAF_AUTO(graph_utils,
655-
object_manager_.GetObject<PropertyGraphUtils>(type_sig));
656-
std::string dst_graph_name = "graph_" + generateId();
657-
BOOST_LEAF_AUTO(dst_wrapper, graph_utils->AddEdgesToGraph(
633+
BOOST_LEAF_AUTO(dst_wrapper, graph_utils->AddLabelsToGraph(
658634
src_frag_id, comm_spec_, *client_,
659635
dst_graph_name, params));
660636
BOOST_LEAF_CHECK(object_manager_.PutObject(dst_wrapper));
@@ -819,13 +795,8 @@ bl::result<std::shared_ptr<DispatchResult>> GrapeInstance::OnReceive(
819795
r->set_graph_def(graph_def);
820796
break;
821797
}
822-
case rpc::ADD_VERTICES: {
823-
BOOST_LEAF_AUTO(graph_def, addVertices(params));
824-
r->set_graph_def(graph_def);
825-
break;
826-
}
827-
case rpc::ADD_EDGES: {
828-
BOOST_LEAF_AUTO(graph_def, addEdges(params));
798+
case rpc::ADD_LABELS: {
799+
BOOST_LEAF_AUTO(graph_def, addLabelsToGraph(params));
829800
r->set_graph_def(graph_def);
830801
break;
831802
}

analytical_engine/core/grape_instance.h

+1-3
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,7 @@ class GrapeInstance : public Subscriber {
118118

119119
bl::result<rpc::GraphDef> copyGraph(const rpc::GSParams& params);
120120

121-
bl::result<rpc::GraphDef> addVertices(const rpc::GSParams& params);
122-
123-
bl::result<rpc::GraphDef> addEdges(const rpc::GSParams& params);
121+
bl::result<rpc::GraphDef> addLabelsToGraph(const rpc::GSParams& params);
124122

125123
bl::result<std::shared_ptr<grape::InArchive>> graphToNumpy(
126124
const rpc::GSParams& params);

analytical_engine/core/loader/arrow_fragment_loader.h

+104-11
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,107 @@ class ArrowFragmentLoader {
140140
return e_tables;
141141
}
142142

143-
boost::leaf::result<vineyard::ObjectID> AddVertices(
143+
boost::leaf::result<vineyard::ObjectID> AddLabelsToGraph(
144+
vineyard::ObjectID frag_id) {
145+
if (!graph_info_->vertices.empty() && !graph_info_->edges.empty()) {
146+
return addVerticesAndEdges(frag_id);
147+
} else if (!graph_info_->vertices.empty()) {
148+
return addVertices(frag_id);
149+
} else {
150+
return addEdges(frag_id);
151+
}
152+
return vineyard::InvalidObjectID();
153+
}
154+
155+
boost::leaf::result<vineyard::ObjectID> addVerticesAndEdges(
156+
vineyard::ObjectID frag_id) {
157+
BOOST_LEAF_AUTO(partitioner, initPartitioner());
158+
BOOST_LEAF_AUTO(partial_v_tables, LoadVertexTables());
159+
BOOST_LEAF_AUTO(partial_e_tables, LoadEdgeTables());
160+
161+
auto basic_fragment_loader = std::make_shared<
162+
vineyard::BasicEVFragmentLoader<OID_T, VID_T, partitioner_t>>(
163+
client_, comm_spec_, partitioner, directed_, true, generate_eid_);
164+
auto frag = std::static_pointer_cast<vineyard::ArrowFragment<oid_t, vid_t>>(
165+
client_.GetObject(frag_id));
166+
for (auto table : partial_v_tables) {
167+
auto meta = table->schema()->metadata();
168+
if (meta == nullptr) {
169+
RETURN_GS_ERROR(vineyard::ErrorCode::kInvalidValueError,
170+
"Metadata of input vertex tables shouldn't be empty.");
171+
}
172+
int label_meta_index = meta->FindKey(LABEL_TAG);
173+
if (label_meta_index == -1) {
174+
RETURN_GS_ERROR(
175+
vineyard::ErrorCode::kInvalidValueError,
176+
"Metadata of input vertex tables should contain label name.");
177+
}
178+
std::string label_name = meta->value(label_meta_index);
179+
BOOST_LEAF_CHECK(
180+
basic_fragment_loader->AddVertexTable(label_name, table));
181+
}
182+
partial_v_tables.clear();
183+
auto old_vm_ptr = frag->GetVertexMap();
184+
BOOST_LEAF_CHECK(
185+
basic_fragment_loader->ConstructVertices(old_vm_ptr->id()));
186+
187+
label_id_t pre_label_num = old_vm_ptr->label_num();
188+
auto schema = frag->schema();
189+
std::map<std::string, label_id_t> vertex_label_to_index;
190+
for (auto& entry : schema.vertex_entries()) {
191+
vertex_label_to_index[entry.label] = entry.id;
192+
}
193+
auto new_labels_index = basic_fragment_loader->get_vertex_label_to_index();
194+
for (auto& pair : new_labels_index) {
195+
vertex_label_to_index[pair.first] = pair.second + pre_label_num;
196+
}
197+
basic_fragment_loader->set_vertex_label_to_index(
198+
std::move(vertex_label_to_index));
199+
for (auto& table_vec : partial_e_tables) {
200+
for (auto table : table_vec) {
201+
auto meta = table->schema()->metadata();
202+
if (meta == nullptr) {
203+
RETURN_GS_ERROR(vineyard::ErrorCode::kInvalidValueError,
204+
"Metadata of input edge tables shouldn't be empty.");
205+
}
206+
207+
int label_meta_index = meta->FindKey(LABEL_TAG);
208+
if (label_meta_index == -1) {
209+
RETURN_GS_ERROR(
210+
vineyard::ErrorCode::kInvalidValueError,
211+
"Metadata of input edge tables should contain label name.");
212+
}
213+
std::string label_name = meta->value(label_meta_index);
214+
215+
int src_label_meta_index = meta->FindKey(SRC_LABEL_TAG);
216+
if (src_label_meta_index == -1) {
217+
RETURN_GS_ERROR(
218+
vineyard::ErrorCode::kInvalidValueError,
219+
"Metadata of input edge tables should contain src label name.");
220+
}
221+
std::string src_label_name = meta->value(src_label_meta_index);
222+
223+
int dst_label_meta_index = meta->FindKey(DST_LABEL_TAG);
224+
if (dst_label_meta_index == -1) {
225+
RETURN_GS_ERROR(
226+
vineyard::ErrorCode::kInvalidValueError,
227+
"Metadata of input edge tables should contain dst label name.");
228+
}
229+
std::string dst_label_name = meta->value(dst_label_meta_index);
230+
231+
BOOST_LEAF_CHECK(basic_fragment_loader->AddEdgeTable(
232+
src_label_name, dst_label_name, label_name, table));
233+
}
234+
}
235+
236+
partial_e_tables.clear();
237+
238+
BOOST_LEAF_CHECK(basic_fragment_loader->ConstructEdges(
239+
schema.all_edge_label_num(), schema.all_vertex_label_num()));
240+
return basic_fragment_loader->AddVerticesAndEdgesToFragment(frag);
241+
}
242+
243+
boost::leaf::result<vineyard::ObjectID> addVertices(
144244
vineyard::ObjectID frag_id) {
145245
BOOST_LEAF_AUTO(partitioner, initPartitioner());
146246
BOOST_LEAF_AUTO(partial_v_tables, LoadVertexTables());
@@ -175,7 +275,7 @@ class ArrowFragmentLoader {
175275
return basic_fragment_loader->AddVerticesToFragment(frag);
176276
}
177277

178-
boost::leaf::result<vineyard::ObjectID> AddEdges(vineyard::ObjectID frag_id) {
278+
boost::leaf::result<vineyard::ObjectID> addEdges(vineyard::ObjectID frag_id) {
179279
BOOST_LEAF_AUTO(partitioner, initPartitioner());
180280
BOOST_LEAF_AUTO(partial_e_tables, LoadEdgeTables());
181281

@@ -366,16 +466,9 @@ class ArrowFragmentLoader {
366466
}
367467
}
368468

369-
boost::leaf::result<vineyard::ObjectID> AddVerticesAsFragmentGroup(
370-
vineyard::ObjectID frag_id) {
371-
BOOST_LEAF_AUTO(new_frag_id, AddVertices(frag_id));
372-
VY_OK_OR_RAISE(client_.Persist(new_frag_id));
373-
return vineyard::ConstructFragmentGroup(client_, new_frag_id, comm_spec_);
374-
}
375-
376-
boost::leaf::result<vineyard::ObjectID> AddEdgesAsFragmentGroup(
469+
boost::leaf::result<vineyard::ObjectID> AddLabelsToGraphAsFragmentGroup(
377470
vineyard::ObjectID frag_id) {
378-
BOOST_LEAF_AUTO(new_frag_id, AddEdges(frag_id));
471+
BOOST_LEAF_AUTO(new_frag_id, AddLabelsToGraph(frag_id));
379472
VY_OK_OR_RAISE(client_.Persist(new_frag_id));
380473
return vineyard::ConstructFragmentGroup(client_, new_frag_id, comm_spec_);
381474
}

analytical_engine/core/object/graph_utils.h

+8-32
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,7 @@ typedef void LoadGraphT(
3737
const std::string& graph_name, const rpc::GSParams& params,
3838
bl::result<std::shared_ptr<IFragmentWrapper>>& fragment_wrapper);
3939

40-
typedef void AddVerticesToGraphT(
41-
vineyard::ObjectID frag_id, const grape::CommSpec& comm_spec,
42-
vineyard::Client& client, const std::string& graph_name,
43-
const rpc::GSParams& params,
44-
bl::result<std::shared_ptr<IFragmentWrapper>>& fragment_wrapper);
45-
46-
typedef void AddEdgesToGraphT(
40+
typedef void AddLabelsToGraphT(
4741
vineyard::ObjectID frag_id, const grape::CommSpec& comm_spec,
4842
vineyard::Client& client, const std::string& graph_name,
4943
const rpc::GSParams& params,
@@ -73,8 +67,7 @@ class PropertyGraphUtils : public GSObject {
7367
lib_path_(std::move(lib_path)),
7468
dl_handle_(nullptr),
7569
load_graph_(nullptr),
76-
add_vertices_to_graph_(nullptr),
77-
add_edges_to_graph_(nullptr),
70+
add_labels_to_graph_(nullptr),
7871
to_arrow_fragment_(nullptr),
7972
to_dynamic_fragment_(nullptr) {}
8073

@@ -84,15 +77,10 @@ class PropertyGraphUtils : public GSObject {
8477
BOOST_LEAF_AUTO(p_fun, get_func_ptr(lib_path_, dl_handle_, "LoadGraph"));
8578
load_graph_ = reinterpret_cast<LoadGraphT*>(p_fun);
8679
}
87-
{
88-
BOOST_LEAF_AUTO(
89-
p_fun, get_func_ptr(lib_path_, dl_handle_, "AddVerticesToGraph"));
90-
add_vertices_to_graph_ = reinterpret_cast<AddVerticesToGraphT*>(p_fun);
91-
}
9280
{
9381
BOOST_LEAF_AUTO(p_fun,
94-
get_func_ptr(lib_path_, dl_handle_, "AddEdgesToGraph"));
95-
add_edges_to_graph_ = reinterpret_cast<AddEdgesToGraphT*>(p_fun);
82+
get_func_ptr(lib_path_, dl_handle_, "AddLabelsToGraph"));
83+
add_labels_to_graph_ = reinterpret_cast<AddLabelsToGraphT*>(p_fun);
9684
}
9785
{
9886
BOOST_LEAF_AUTO(p_fun,
@@ -116,25 +104,14 @@ class PropertyGraphUtils : public GSObject {
116104
return wrapper;
117105
}
118106

119-
bl::result<std::shared_ptr<IFragmentWrapper>> AddVerticesToGraph(
120-
vineyard::ObjectID frag_id, const grape::CommSpec& comm_spec,
121-
vineyard::Client& client, const std::string& graph_name,
122-
const rpc::GSParams& params) {
123-
bl::result<std::shared_ptr<IFragmentWrapper>> wrapper;
124-
125-
add_vertices_to_graph_(frag_id, comm_spec, client, graph_name, params,
126-
wrapper);
127-
return wrapper;
128-
}
129-
130-
bl::result<std::shared_ptr<IFragmentWrapper>> AddEdgesToGraph(
107+
bl::result<std::shared_ptr<IFragmentWrapper>> AddLabelsToGraph(
131108
vineyard::ObjectID frag_id, const grape::CommSpec& comm_spec,
132109
vineyard::Client& client, const std::string& graph_name,
133110
const rpc::GSParams& params) {
134111
bl::result<std::shared_ptr<IFragmentWrapper>> wrapper;
135112

136-
add_edges_to_graph_(frag_id, comm_spec, client, graph_name, params,
137-
wrapper);
113+
add_labels_to_graph_(frag_id, comm_spec, client, graph_name, params,
114+
wrapper);
138115
return wrapper;
139116
}
140117

@@ -172,8 +149,7 @@ class PropertyGraphUtils : public GSObject {
172149
std::string lib_path_;
173150
void* dl_handle_;
174151
LoadGraphT* load_graph_;
175-
AddVerticesToGraphT* add_vertices_to_graph_;
176-
AddEdgesToGraphT* add_edges_to_graph_;
152+
AddLabelsToGraphT* add_labels_to_graph_;
177153
ToArrowFragmentT* to_arrow_fragment_;
178154
ToDynamicFragmentT* to_dynamic_fragment_;
179155
};

0 commit comments

Comments
 (0)