From 69abf33029aa65275d9b0fae414a4e8ec709d7ff Mon Sep 17 00:00:00 2001 From: abaybektursun Date: Mon, 16 Oct 2017 17:11:25 -0500 Subject: [PATCH 1/5] python 3 and graphx 2.0 support --- gem/embedding/gf.py | 19 ++++++++++--------- gem/embedding/hope.py | 11 ++++++----- gem/embedding/lap.py | 9 +++++---- gem/embedding/lle.py | 9 +++++---- gem/embedding/node2vec.py | 10 ++++++---- gem/embedding/sdne.py | 15 ++++++++------- gem/evaluation/metrics.py | 8 ++++---- gem/evaluation/visualize_embedding.py | 4 ++-- gem/utils/evaluation_util.py | 6 +++--- gem/utils/graph_util.py | 7 ++++--- gem/version.py | 11 +++++++++++ 11 files changed, 64 insertions(+), 45 deletions(-) create mode 100644 gem/version.py diff --git a/gem/embedding/gf.py b/gem/embedding/gf.py index 82188ed..0e9d762 100644 --- a/gem/embedding/gf.py +++ b/gem/embedding/gf.py @@ -1,6 +1,6 @@ disp_avlbl = True -from os import environ -if 'DISPLAY' not in environ: +import os +if 'DISPLAY' not in os.environ: disp_avlbl = False import matplotlib matplotlib.use('Agg') @@ -12,8 +12,9 @@ import sys sys.path.append('./') +sys.path.append(os.path.realpath(__file__)) -from static_graph_embedding import StaticGraphEmbedding +from .static_graph_embedding import StaticGraphEmbedding from gem.utils import graph_util, plot_util from gem.evaluation import visualize_embedding as viz from time import time @@ -43,7 +44,7 @@ def get_method_summary(self): def _get_f_value(self, graph): f1 = 0 - for i, j, w in graph.edges_iter(data='weight', default=1): + for i, j, w in graph.edges(data='weight', default=1): f1 += (w - np.dot(self._X[i, :], self._X[j, :]))**2 f2 = self._regu*(np.linalg.norm(self._X)**2) return [f1, f2, f1+f2] @@ -56,7 +57,7 @@ def learn_embedding(self, graph=None, edge_f=None, is_weighted=False, no_python= try: from c_ext import graphFac_ext except: - print 'Could not import C++ module for Graph Factorization. Reverting to python implementation. Please recompile graphFac_ext from graphFac.cpp using bjam' + print('Could not import C++ module for Graph Factorization. Reverting to python implementation. Please recompile graphFac_ext from graphFac.cpp using bjam') c_flag = False if c_flag: if edge_f: @@ -78,9 +79,9 @@ def learn_embedding(self, graph=None, edge_f=None, is_weighted=False, no_python= for iter_id in range(self._max_iter): if not iter_id%100: [f1, f2, f] = self._get_f_value(graph) - print '\t\tIter id: %d, Objective value: %g, f1: %g, f2: %g' % (iter_id, f, f1, f2) + print('\t\tIter id: %d, Objective value: %g, f1: %g, f2: %g' % (iter_id, f, f1, f2)) tempFlag = False - for i, j, w in graph.edges_iter(data='weight', default=1): + for i, j, w in graph.edges(data='weight', default=1): if j <= i: continue delPhi = -(w - np.dot(self._X[i, :], self._X[j, :]))*self._X[j, :] + self._regu*self._X[i, :] @@ -114,11 +115,11 @@ def get_reconstructed_adj(self, X=None, node_l=None): G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=False) G = G.to_directed() res_pre = 'results/testKarate' - print 'Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges()) + print ('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) t1 = time() embedding = GraphFactorization(2, 100000, 1*10**-4, 1.0) embedding.learn_embedding(graph=G, edge_f=None, is_weighted=True, no_python=True) - print 'Graph Factorization:\n\tTraining time: %f' % (time() - t1) + print ('Graph Factorization:\n\tTraining time: %f' % (time() - t1)) viz.plot_embedding2D(embedding.get_embedding(), di_graph=G, node_colors=None) plt.show() diff --git a/gem/embedding/hope.py b/gem/embedding/hope.py index 565097d..0098afa 100644 --- a/gem/embedding/hope.py +++ b/gem/embedding/hope.py @@ -1,6 +1,6 @@ disp_avlbl = True -from os import environ -if 'DISPLAY' not in environ: +import os +if 'DISPLAY' not in os.environ: disp_avlbl = False import matplotlib matplotlib.use('Agg') @@ -16,6 +16,7 @@ import sys sys.path.append('./') +sys.path.append(os.path.realpath(__file__)) from static_graph_embedding import StaticGraphEmbedding from gem.utils import graph_util, plot_util @@ -59,7 +60,7 @@ def learn_embedding(self, graph=None, edge_f=None, is_weighted=False, no_python= p_d_p_t = np.dot(u, np.dot(np.diag(s), vt)) eig_err = np.linalg.norm(p_d_p_t - S) - print 'SVD error (low rank): %f' % eig_err + print('SVD error (low rank): %f' % eig_err) # p_d_p_t = np.dot(self._X, np.dot(w[1:self._d+1, 1:self._d+1], self._X.T)) # eig_err = np.linalg.norm(p_d_p_t - L_sym) @@ -92,11 +93,11 @@ def get_reconstructed_adj(self, X=None, node_l=None): G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=False) G = G.to_directed() res_pre = 'results/testKarate' - print 'Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges()) + print('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) t1 = time() embedding = HOPE(4, 0.01) embedding.learn_embedding(graph=G, edge_f=None, is_weighted=True, no_python=True) - print 'HOPE:\n\tTraining time: %f' % (time() - t1) + print('HOPE:\n\tTraining time: %f' % (time() - t1)) viz.plot_embedding2D(embedding.get_embedding()[:, :2], di_graph=G, node_colors=None) plt.show() diff --git a/gem/embedding/lap.py b/gem/embedding/lap.py index 1ffd9d1..8180964 100644 --- a/gem/embedding/lap.py +++ b/gem/embedding/lap.py @@ -1,6 +1,6 @@ disp_avlbl = True -from os import environ -if 'DISPLAY' not in environ: +import os +if 'DISPLAY' not in os.environ: disp_avlbl = False import matplotlib matplotlib.use('Agg') @@ -16,6 +16,7 @@ import sys sys.path.append('./') +sys.path.append(os.path.realpath(__file__)) from static_graph_embedding import StaticGraphEmbedding from gem.utils import graph_util, plot_util @@ -95,11 +96,11 @@ def get_reconstructed_adj(self, X=None, node_l=None): G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=False) G = G.to_directed() res_pre = 'results/testKarate' - print 'Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges()) + print('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) t1 = time() embedding = LaplacianEigenmaps(2) embedding.learn_embedding(graph=G, edge_f=None, is_weighted=True, no_python=True) - print 'Laplacian Eigenmaps:\n\tTraining time: %f' % (time() - t1) + print('Laplacian Eigenmaps:\n\tTraining time: %f' % (time() - t1))1 viz.plot_embedding2D(embedding.get_embedding(), di_graph=G, node_colors=None) plt.show() diff --git a/gem/embedding/lle.py b/gem/embedding/lle.py index 3b92dd9..e291c30 100644 --- a/gem/embedding/lle.py +++ b/gem/embedding/lle.py @@ -1,6 +1,6 @@ disp_avlbl = True -from os import environ -if 'DISPLAY' not in environ: +import os +if 'DISPLAY' not in os.environ: disp_avlbl = False import matplotlib matplotlib.use('Agg') @@ -18,6 +18,7 @@ import sys sys.path.append('./') +sys.path.append(os.path.realpath(__file__)) from static_graph_embedding import StaticGraphEmbedding from gem.utils import graph_util, plot_util @@ -79,11 +80,11 @@ def get_reconstructed_adj(self, X=None, node_l=None): G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=False) G = G.to_directed() res_pre = 'results/testKarate' - print 'Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges()) + print('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) t1 = time() embedding = LocallyLinearEmbedding(2) embedding.learn_embedding(graph=G, edge_f=None, is_weighted=True, no_python=True) - print 'Graph Factorization:\n\tTraining time: %f' % (time() - t1) + print('Graph Factorization:\n\tTraining time: %f' % (time() - t1)) viz.plot_embedding2D(embedding.get_embedding(), di_graph=G, node_colors=None) plt.show() diff --git a/gem/embedding/node2vec.py b/gem/embedding/node2vec.py index bb27ae8..3506518 100644 --- a/gem/embedding/node2vec.py +++ b/gem/embedding/node2vec.py @@ -1,6 +1,6 @@ disp_avlbl = True -from os import environ -if 'DISPLAY' not in environ: +import os +if 'DISPLAY' not in os.environ: disp_avlbl = False import matplotlib matplotlib.use('Agg') @@ -16,6 +16,8 @@ import sys sys.path.append('./') +sys.path.append(os.path.realpath(__file__)) + from subprocess import call from static_graph_embedding import StaticGraphEmbedding @@ -96,11 +98,11 @@ def get_reconstructed_adj(self, X=None, node_l=None): G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=False) G = G.to_directed() res_pre = 'results/testKarate' - print 'Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges()) + print('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) t1 = time() embedding = node2vec(2, 1, 80, 10, 10, 1, 1) embedding.learn_embedding(graph=G, edge_f=None, is_weighted=True, no_python=True) - print 'node2vec:\n\tTraining time: %f' % (time() - t1) + print('node2vec:\n\tTraining time: %f' % (time() - t1)) viz.plot_embedding2D(embedding.get_embedding(), di_graph=G, node_colors=None) plt.show() diff --git a/gem/embedding/sdne.py b/gem/embedding/sdne.py index aadd67d..b7a8850 100644 --- a/gem/embedding/sdne.py +++ b/gem/embedding/sdne.py @@ -1,6 +1,6 @@ disp_avlbl = True -from os import environ -if 'DISPLAY' not in environ: +import os +if 'DISPLAY' not in os.environ: disp_avlbl = False import matplotlib matplotlib.use('Agg') @@ -11,6 +11,7 @@ import sys sys.path.append('./') +sys.path.append(os.path.realpath(__file__)) from static_graph_embedding import StaticGraphEmbedding from gem.utils import graph_util, plot_util @@ -95,7 +96,7 @@ def learn_embedding(self, graph=None, edge_f=None, is_weighted=False, no_python= else: S = graph_util.transform_DiGraph_to_adj(graph) if not np.allclose(S.T, S): - print "SDNE only works for symmetric graphs! Making the graph symmetric" + print("SDNE only works for symmetric graphs! Making the graph symmetric") t1 = time() S = (S + S.T)/2 # enforce S is symmetric S -= np.diag(np.diag(S)) # enforce diagonal = 0 @@ -159,7 +160,7 @@ def weighted_mse_y(y_true, y_pred): OutData = np.zeros((data_chunk_size, 2*self._node_num + 3)) # Train the model for epoch_num in range(self._num_iter): - print 'EPOCH %d/%d' % (epoch_num, self._num_iter) + print('EPOCH %d/%d' % (epoch_num, self._num_iter)) e = 0 k = 0 for i in range(self._node_num): @@ -254,11 +255,11 @@ def get_reconst_from_embed(self, embed, node_l=None, filesuffix=None): G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=False) G = G.to_directed() res_pre = 'results/testKarate' - print 'Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges()) + print('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) t1 = time() embedding = SDNE(d=2, beta=5, alpha=1e-5, nu1=1e-6, nu2=1e-6, K=3, n_units=[50, 15,], rho=0.3, n_iter=50, xeta=0.01, n_batch=500, modelfile=['./intermediate/enc_model.json', './intermediate/dec_model.json'], weightfile=['./intermediate/enc_weights.hdf5', './intermediate/dec_weights.hdf5']) embedding.learn_embedding(graph=G, edge_f=None, is_weighted=True, no_python=True) - print 'SDNE:\n\tTraining time: %f' % (time() - t1) + print('SDNE:\n\tTraining time: %f' % (time() - t1)) viz.plot_embedding2D(embedding.get_embedding(), di_graph=G, node_colors=None) - plt.show() \ No newline at end of file + plt.show() diff --git a/gem/evaluation/metrics.py b/gem/evaluation/metrics.py index e04b485..5ab0cc4 100644 --- a/gem/evaluation/metrics.py +++ b/gem/evaluation/metrics.py @@ -13,7 +13,7 @@ def computePrecisionCurve(predicted_edge_list, true_digraph, max_k=-1): precision_scores = [] delta_factors = [] correct_edge = 0 - for i in xrange(max_k): + for i in range(max_k): if true_digraph.has_edge(sorted_edges[i][0], sorted_edges[i][1]): correct_edge += 1 delta_factors.append(1.0) @@ -25,13 +25,13 @@ def computePrecisionCurve(predicted_edge_list, true_digraph, max_k=-1): def computeMAP(predicted_edge_list, true_digraph, max_k=-1): node_num = true_digraph.number_of_nodes() node_edges = [] - for i in xrange(node_num): + for i in range(node_num): node_edges.append([]) for (st, ed, w) in predicted_edge_list: node_edges[st].append((st, ed, w)) node_AP = [0.0] * node_num count = 0 - for i in xrange(node_num): + for i in range(node_num): if true_digraph.out_degree(i) == 0: continue count += 1 @@ -75,4 +75,4 @@ def getNodeAnomaly(X_dyn): node_anom = np.zeros((n_nodes, T-1)) for t in range(T-1): node_anom[:, t] = np.linalg.norm(X_dyn[t+1][:n_nodes, :] - X_dyn[t][:n_nodes, :], axis = 1) - return node_anom \ No newline at end of file + return node_anom diff --git a/gem/evaluation/visualize_embedding.py b/gem/evaluation/visualize_embedding.py index 47677a8..9df5997 100644 --- a/gem/evaluation/visualize_embedding.py +++ b/gem/evaluation/visualize_embedding.py @@ -5,7 +5,7 @@ def plot_embedding2D(node_pos, node_colors=None, di_graph=None): node_num, embedding_dimension = node_pos.shape if(embedding_dimension > 2): - print "Embedding dimensiion greater than 2, use tSNE to reduce it to 2" + print("Embedding dimensiion greater than 2, use tSNE to reduce it to 2") model = TSNE(n_components=2) node_pos = model.fit_transform(node_pos) @@ -15,7 +15,7 @@ def plot_embedding2D(node_pos, node_colors=None, di_graph=None): else: # plot using networkx with edge structure pos = {} - for i in xrange(node_num): + for i in range(node_num): pos[i] = node_pos[i, :] if node_colors: nx.draw_networkx_nodes(di_graph, pos, node_color=node_colors, width=0.1, node_size=100, arrows=False, alpha=0.8, font_size=5) diff --git a/gem/utils/evaluation_util.py b/gem/utils/evaluation_util.py index ca121c4..aeeeffd 100644 --- a/gem/utils/evaluation_util.py +++ b/gem/utils/evaluation_util.py @@ -23,8 +23,8 @@ def getEdgeListFromAdjMtx(adj, threshold=0.0, is_undirected=True, edge_pairs=Non if adj[st, ed] >= threshold: result.append((st, ed, adj[st, ed])) else: - for i in xrange(node_num): - for j in xrange(node_num): + for i in range(node_num): + for j in range(node_num): if(j == i): continue if(is_undirected and i >= j): @@ -48,4 +48,4 @@ def splitDiGraphToTrainTest(di_graph, train_ratio, is_undirected=True): train_digraph.remove_edge(st, ed) if(is_undirected): train_digraph.remove_edge(ed, st) - return (train_digraph, test_digraph) \ No newline at end of file + return (train_digraph, test_digraph) diff --git a/gem/utils/graph_util.py b/gem/utils/graph_util.py index ae1710c..120e06a 100644 --- a/gem/utils/graph_util.py +++ b/gem/utils/graph_util.py @@ -1,4 +1,5 @@ -import cPickle as pickle +try: import cPickle as pickle +except: import pickle import numpy as np import networkx as nx import random @@ -88,7 +89,7 @@ def addNodeAnomalies(di_graphs, p, k): # pdb.set_trace() di_graphs[t].add_edges_from(itertools.product(list(anomalous_nodes), range(n_nodes))) di_graphs[t].add_edges_from(itertools.product(range(n_nodes), list(anomalous_nodes))) - print 'Nodes: %d, Edges: %d' % (di_graphs[t].number_of_nodes(), di_graphs[t].number_of_edges()) + print('Nodes: %d, Edges: %d' % (di_graphs[t].number_of_nodes(), di_graphs[t].number_of_edges())) return anomaly_time_steps def saveGraphToEdgeListTxt(graph, file_name): @@ -183,4 +184,4 @@ def saveDynamicSBmGraph(file_perfix, dynamic_graphs): node_infos = {} node_infos['community'] = dynamic_graphs[i][1] node_infos['perturbation'] = dynamic_graphs[i][2] - pickle.dump(node_infos, fp) \ No newline at end of file + pickle.dump(node_infos, fp) diff --git a/gem/version.py b/gem/version.py new file mode 100644 index 0000000..81e1bd7 --- /dev/null +++ b/gem/version.py @@ -0,0 +1,11 @@ + + # THIS FILE IS GENERATED FROM GEM SETUP.PY + short_version = '1.0.0' + version = '1.0.0' + full_version = '1.0.0' + git_revision = '4f87b50c41079948077a572482e8fd85da12f599' + release = True + if not release: + version = full_version + short_version += ".dev" + \ No newline at end of file From 422883c205323958a73196932ee28a8b2475e722 Mon Sep 17 00:00:00 2001 From: abaybektursun Date: Mon, 16 Oct 2017 17:22:08 -0500 Subject: [PATCH 2/5] readme and ignore gem/version.py --- .gitignore | 5 ++++- README.md | 8 ++++++-- gem/version.py | 11 ----------- 3 files changed, 10 insertions(+), 14 deletions(-) delete mode 100644 gem/version.py diff --git a/.gitignore b/.gitignore index a11e1fb..da69ba7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# THIS FILE IS GENERATED FROM GEM SETUP.PY +gem/version.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -101,4 +104,4 @@ gem/intermediate/* *.json # images -*.png \ No newline at end of file +*.png diff --git a/README.md b/README.md index e826956..163e3c5 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,16 @@ To run SDNE, GEM requires Theano >= 0.9.0 and Keras = 2.0.2. ## Install The package uses setuptools, which is a common way of installing python modules. To install in your home directory, use: - +```bash python setup.py install --user +``` To install for all users on Unix/Linux: - +```bash sudo python setup.py install +``` + +You also can use `python3` instead of `python` ## Usage Run Graph Factorization on Karate graph and evaluate it on graph reconstruction: diff --git a/gem/version.py b/gem/version.py deleted file mode 100644 index 81e1bd7..0000000 --- a/gem/version.py +++ /dev/null @@ -1,11 +0,0 @@ - - # THIS FILE IS GENERATED FROM GEM SETUP.PY - short_version = '1.0.0' - version = '1.0.0' - full_version = '1.0.0' - git_revision = '4f87b50c41079948077a572482e8fd85da12f599' - release = True - if not release: - version = full_version - short_version += ".dev" - \ No newline at end of file From f8d647cd5441684303f20f057194d14b72bf05d5 Mon Sep 17 00:00:00 2001 From: abaybektursun Date: Tue, 17 Oct 2017 00:48:40 -0500 Subject: [PATCH 3/5] porintg other models --- .gitignore | 3 +++ README.md | 4 +++- gem/embedding/.hope.py.swp | Bin 0 -> 12288 bytes gem/embedding/hope.py | 2 +- gem/embedding/lap.py | 2 +- gem/embedding/lle.py | 2 +- gem/embedding/node2vec.py | 2 +- gem/embedding/sdne.py | 14 +++++++++----- gem/utils/graph_util.py | 8 ++++---- test.py | 29 +++++++++++++++++++++++++++++ 10 files changed, 52 insertions(+), 14 deletions(-) create mode 100644 gem/embedding/.hope.py.swp create mode 100644 test.py diff --git a/.gitignore b/.gitignore index da69ba7..aeb4d2d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +py2test/ +py3test/ + # THIS FILE IS GENERATED FROM GEM SETUP.PY gem/version.py diff --git a/README.md b/README.md index 163e3c5..393e05a 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,14 @@ The graphs are saved using `nx.write_gpickle` in the networkx format and can be * **gem/c_ext**: Python interface for source files in c_src using [Boost.Python](http://www.boost.org/doc/libs/1_64_0/libs/python/doc/html/index.html) ## Dependencies -GEM is tested to work on Python 2.7. +GEM is tested to work on Python 2.7 and Python 3.6 The required dependencies are: Numpy >= 1.12.0, SciPy >= 0.19.0, Networkx >= 1.11, Scikit-learn >= 0.18.1. To run SDNE, GEM requires Theano >= 0.9.0 and Keras = 2.0.2. +In case of Python 3, make sure it was compiled with `./configure --enable-shared`, and that you have `/usr/local/bin/python` in your `LD_LIBRARY_PATH` + ## Install The package uses setuptools, which is a common way of installing python modules. To install in your home directory, use: ```bash diff --git a/gem/embedding/.hope.py.swp b/gem/embedding/.hope.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..3ebfc11bccb4bd928c32f7917b7e4e871eb093ec GIT binary patch literal 12288 zcmeHNO>87b6|Nxu2H3D992Nkb{Sp*SC9JmDuDR2M|KtMro=7K~d?)(Ue0|)_vYq)^8grDzKb(1Rh1u)^H&s)m zK&8N|Qs5GMcztb^hSuNrUjE<%PrNEG)u2j&N`Xp&N`Xp&N`Xp&N`Xp&N`cp=0yevk zJp<~mE9j4k-}}z}7LV0Wr9h=Xr9h=Xr9h=Xr9h=Xr9h=Xr9h=Xr9h>?Yfu3hG4=%h z_FtdD8Tc9SL*O~!bHD+x4}1i8Kky*%^8Jkc33w6s0q_FwE#S+* z0k99$fqz_r4e(Roo4_}KXMm@HM}e!ryMQae%Wq@skH8;*7lE$;&jQzgE5J){W$gFB z_kibtuK_7=6L<)CC-AqoF!mGR`@k20F97?%hk#4KziW*B3iu`PZQ!%OHn0W!>&=Y) z0{9Ye3>*QsfsX=@0`CTX_a?@E3w#H77B~S$z-NGu0e^oZW4{4@415=O4)`?i81M<8 z4O|BP`Ub{+1biKs0V}{2;Gf9J3%~@hzpr{{ITpve<@Zl%51lR-5WO zVHT%W%saptZl|h5A@PHnICcnzJ;`93}wyXhEE zjNlHB6wy68#@!HYhbkB8K+wER&}{MLMrR&0X*#(wBFhz`G;lkKt!Z0j>|-*QRyA0y zi=J+1QxWN0^)U&`2Q|9ciiy>67^gDolF&Evbi4^FTiV#&QyX|8C=q__#!VR;NMG|E z6f}}nwvS!3kXmC5Mc6tgr~Fg0Ka%;ZVYnycqcT0URxu@f zE0%B>9f`3$6*-z}wfWQDROl9;ilYOBH!2?Z3wSL9es+>R87 znrUKeKB0sZnt)0fxty3wzQA}8Q4rqobyF5S*z}10(wH9ho6OYS>H#`1(H@0MQp;(- zU~HrYoVMDT=mK*H$Y+c9iWTo%#(tMACj>K$QtOOE(c427I-X3_rrlZ)+dC?6(cC#e z63V!4(G)rI5#izpGJ|fB^TNl{nDR?L7xp6C7IyghRqSnesdHBncwWBi})so=~=rxy-UWJ#Ci70vlbx#JMW&eQIN!$aokgX+91z5s6G#RFo}YX5%7o z(dZV!l1x|xUbd6HwNkE!)>?e&VWC~Pi!F`dMPb`{^2T|>=b2LQS3q=!>`LBqijz4x z;;7{?Feo%r>q!*3@bkxd?W3S1KoIo6zqbxkd8*BZQ{Jw_uB>yl9z7NFp4xEN$Dsgd zG*9PPM6P(%dQ=_@!-CBPR_HOr3%1CCnMNkqY4PI?&k)PZiMF$F1&>9-bTSRL#SmP# zMNA^#2X=nsp|bJ=*Inb#4^}hfkAb@@Fr6UhhFzyYr&L_AN3$7mWh0*HZ zLh)|h?XiwVu*<^#N(7ASd=cFAHG-Q==0@GcVfg^UxSA{bi1EtHKg|TygbIlppXvFfuoR`^~xa;ML{P=RP!q>64c8IN&r-K**>Ik{^nooh?u7HbBo z(~5(fzKv=)mF9WFl2>#!=-Xv6#By)c1q~D{?uF)zj|hg5ryvAgeyk8KHqPvpi$*20 zA4_BWo5_xqR`~uwnY5!2#abm%kQR@*k}RL}rLlh4Z3*WKhdv&^_ZoEXZ|7pN+EsdTS6*R67hnP3@6mwq~xYkIU vOvq_`NY&8o8hR;f+_<%Kd+YjB4Yy12Mg*x Date: Sun, 22 Oct 2017 13:01:05 -0500 Subject: [PATCH 4/5] tested --- .gitignore | 5 +++ README.md | 66 ++++++++++++++++++++++++++++--------- gem/c_exe/readme.txt | 3 +- gem/embedding/.hope.py.swp | Bin 12288 -> 0 bytes gem/embedding/gf.py | 6 ++-- gem/embedding/hope.py | 4 +-- gem/embedding/lap.py | 2 +- gem/embedding/node2vec.py | 7 ++-- gem/embedding/sdne.py | 3 +- test.py | 61 ++++++++++++++++++++++++---------- 10 files changed, 113 insertions(+), 44 deletions(-) delete mode 100644 gem/embedding/.hope.py.swp diff --git a/.gitignore b/.gitignore index aeb4d2d..76298df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +intermediate/ +.idea +marvel_* +tempGraph.* +ES_EVENT_LOG_v11.* py2test/ py3test/ diff --git a/README.md b/README.md index 393e05a..7af3e95 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ GEM implements the following graph embedding techniques: * [Laplacian Eigenmaps](http://yeolab.weebly.com/uploads/2/5/5/0/25509700/belkin_laplacian_2003.pdf) * [Locally Linear Embedding](http://www.robots.ox.ac.uk/~az/lectures/ml/lle.pdf) * [Graph Factorization](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/40839.pdf) -* [Higher-Prder Proximity preserved Embedding (HOPE)](http://www.kdd.org/kdd2016/papers/files/rfp0184-ouA.pdf) +* [Higher-Order Proximity preserved Embedding (HOPE)](http://www.kdd.org/kdd2016/papers/files/rfp0184-ouA.pdf) * [Structural Deep Network Embedding (SDNE)](http://www.kdd.org/kdd2016/papers/files/rfp0191-wangAemb.pdf) * [node2vec](http://www.kdd.org/kdd2016/papers/files/rfp0218-groverA.pdf) @@ -50,23 +50,57 @@ To install for all users on Unix/Linux: You also can use `python3` instead of `python` ## Usage -Run Graph Factorization on Karate graph and evaluate it on graph reconstruction: - - from gem.embedding.gf import GraphFactorization as gf - from gem.evaluation import evaluate_graph_reconstruction as gr - from gem.utils import graph_util - - # Instatiate the embedding method with hyperparameters - em = gf(2, 100000, 1*10**-4, 1.0) - - # Load graph - graph = graph_util.loadGraphFromEdgeListTxt('gem/data/karate.edgelist') - +Run the methods on Karate graph and evaluate them on graph reconstruction: + +```python +import matplotlib.pyplot as plt + +from gem.utils import graph_util, plot_util +from gem.evaluation import visualize_embedding as viz +from gem.evaluation import evaluate_graph_reconstruction as gr +from time import time + +from gem.embedding.gf import GraphFactorization +from gem.embedding.hope import HOPE +from gem.embedding.lap import LaplacianEigenmaps +from gem.embedding.lle import LocallyLinearEmbedding +from gem.embedding.node2vec import node2vec +from gem.embedding.sdne import SDNE + +# File that contains the edges. Format: source target +# Optionally, you can add weights as third column: source target weight +edge_f = 'gem/data/karate.edgelist' +# Specify whether the edges are directed +isDirected = True + +# Load graph +G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=isDirected) +G = G.to_directed() + +models = [] +# You can comment out the methods you don't want to run +models.append(GraphFactorization(2, 100000, 1*10**-4, 1.0)) +models.append(HOPE(4, 0.01)) +models.append(LaplacianEigenmaps(2)) +models.append(LocallyLinearEmbedding(2)) +models.append(node2vec(2, 1, 80, 10, 10, 1, 1)) +models.append(SDNE(d=2, beta=5, alpha=1e-5, nu1=1e-6, nu2=1e-6, K=3,n_units=[50, 15,], rho=0.3, n_iter=50, xeta=0.01,n_batch=500, + modelfile=['./intermediate/enc_model.json', './intermediate/dec_model.json'], + weightfile=['./intermediate/enc_weights.hdf5', './intermediate/dec_weights.hdf5'])) + +for embedding in models: + print ('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) + t1 = time() # Learn embedding - accepts a networkx graph or file with edge list - Y, t = em.learn_embedding(graph, edge_f=None, is_weighted=True, no_python=True) - + Y, t = embedding.learn_embedding(graph=G, edge_f=None, is_weighted=True, no_python=True) + print (embedding._method_name+':\n\tTraining time: %f' % (time() - t1)) # Evaluate on graph reconstruction - MAP, prec_curv = gr.evaluateStaticGraphReconstruction(graph, em, Y, None) + MAP, prec_curv = gr.evaluateStaticGraphReconstruction(G, embedding, Y, None) + # Visualize + viz.plot_embedding2D(embedding.get_embedding(), di_graph=G, node_colors=None) + plt.show() +``` + ## Cite @article{goyal2017graph, diff --git a/gem/c_exe/readme.txt b/gem/c_exe/readme.txt index efefa78..fc117d9 100644 --- a/gem/c_exe/readme.txt +++ b/gem/c_exe/readme.txt @@ -1,2 +1,3 @@ 1. Recompile from https://github.com/snap-stanford/snap and copy node2vec executable to this folder -2. To grant executable permission, run: chmod +x ./c_exe/node2vec \ No newline at end of file +2. To grant executable permission, run: chmod +x ./c_exe/node2vec + diff --git a/gem/embedding/.hope.py.swp b/gem/embedding/.hope.py.swp deleted file mode 100644 index 3ebfc11bccb4bd928c32f7917b7e4e871eb093ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNO>87b6|Nxu2H3D992Nkb{Sp*SC9JmDuDR2M|KtMro=7K~d?)(Ue0|)_vYq)^8grDzKb(1Rh1u)^H&s)m zK&8N|Qs5GMcztb^hSuNrUjE<%PrNEG)u2j&N`Xp&N`Xp&N`Xp&N`Xp&N`cp=0yevk zJp<~mE9j4k-}}z}7LV0Wr9h=Xr9h=Xr9h=Xr9h=Xr9h=Xr9h=Xr9h>?Yfu3hG4=%h z_FtdD8Tc9SL*O~!bHD+x4}1i8Kky*%^8Jkc33w6s0q_FwE#S+* z0k99$fqz_r4e(Roo4_}KXMm@HM}e!ryMQae%Wq@skH8;*7lE$;&jQzgE5J){W$gFB z_kibtuK_7=6L<)CC-AqoF!mGR`@k20F97?%hk#4KziW*B3iu`PZQ!%OHn0W!>&=Y) z0{9Ye3>*QsfsX=@0`CTX_a?@E3w#H77B~S$z-NGu0e^oZW4{4@415=O4)`?i81M<8 z4O|BP`Ub{+1biKs0V}{2;Gf9J3%~@hzpr{{ITpve<@Zl%51lR-5WO zVHT%W%saptZl|h5A@PHnICcnzJ;`93}wyXhEE zjNlHB6wy68#@!HYhbkB8K+wER&}{MLMrR&0X*#(wBFhz`G;lkKt!Z0j>|-*QRyA0y zi=J+1QxWN0^)U&`2Q|9ciiy>67^gDolF&Evbi4^FTiV#&QyX|8C=q__#!VR;NMG|E z6f}}nwvS!3kXmC5Mc6tgr~Fg0Ka%;ZVYnycqcT0URxu@f zE0%B>9f`3$6*-z}wfWQDROl9;ilYOBH!2?Z3wSL9es+>R87 znrUKeKB0sZnt)0fxty3wzQA}8Q4rqobyF5S*z}10(wH9ho6OYS>H#`1(H@0MQp;(- zU~HrYoVMDT=mK*H$Y+c9iWTo%#(tMACj>K$QtOOE(c427I-X3_rrlZ)+dC?6(cC#e z63V!4(G)rI5#izpGJ|fB^TNl{nDR?L7xp6C7IyghRqSnesdHBncwWBi})so=~=rxy-UWJ#Ci70vlbx#JMW&eQIN!$aokgX+91z5s6G#RFo}YX5%7o z(dZV!l1x|xUbd6HwNkE!)>?e&VWC~Pi!F`dMPb`{^2T|>=b2LQS3q=!>`LBqijz4x z;;7{?Feo%r>q!*3@bkxd?W3S1KoIo6zqbxkd8*BZQ{Jw_uB>yl9z7NFp4xEN$Dsgd zG*9PPM6P(%dQ=_@!-CBPR_HOr3%1CCnMNkqY4PI?&k)PZiMF$F1&>9-bTSRL#SmP# zMNA^#2X=nsp|bJ=*Inb#4^}hfkAb@@Fr6UhhFzyYr&L_AN3$7mWh0*HZ zLh)|h?XiwVu*<^#N(7ASd=cFAHG-Q==0@GcVfg^UxSA{bi1EtHKg|TygbIlppXvFfuoR`^~xa;ML{P=RP!q>64c8IN&r-K**>Ik{^nooh?u7HbBo z(~5(fzKv=)mF9WFl2>#!=-Xv6#By)c1q~D{?uF)zj|hg5ryvAgeyk8KHqPvpi$*20 zA4_BWo5_xqR`~uwnY5!2#abm%kQR@*k}RL}rLlh4Z3*WKhdv&^_ZoEXZ|7pN+EsdTS6*R67hnP3@6mwq~xYkIU vOvq_`NY&8o8hR;f+_<%Kd+YjB4Yy12Mg*x Date: Wed, 25 Oct 2017 13:25:23 -0500 Subject: [PATCH 5/5] sync --- .gitignore | 2 ++ gem/embedding/gf.py | 2 +- gem/evaluation/evaluate_graph_reconstruction.py | 4 ++-- gem/utils/graph_util.py | 4 ++-- test.py | 17 +++++++---------- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 76298df..c9d9ee8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.edgelist +*.idx2node intermediate/ .idea marvel_* diff --git a/gem/embedding/gf.py b/gem/embedding/gf.py index 30bbf37..e4c6ed1 100644 --- a/gem/embedding/gf.py +++ b/gem/embedding/gf.py @@ -68,7 +68,7 @@ def learn_embedding(self, graph=None, edge_f=None, is_weighted=False, no_python= is_weighted = True edge_f = 'tempGraph.graph' t1 = time() - graphFac_ext.learn_embedding(edge_f, "tempGraphGF.emb", True, is_weighted, self._d, self._eta, self._regu, self._max_iter) + graphFac_ext.learn_embedding(edge_f, "tempGraphGF.emb", True, is_weighted, self._d, self._eta, self._regu, self._max_iter) self._X = graph_util.loadEmbedding('tempGraphGF.emb') t2 = time() return self._X, (t2-t1) diff --git a/gem/evaluation/evaluate_graph_reconstruction.py b/gem/evaluation/evaluate_graph_reconstruction.py index 4245d34..c186a9e 100644 --- a/gem/evaluation/evaluate_graph_reconstruction.py +++ b/gem/evaluation/evaluate_graph_reconstruction.py @@ -14,10 +14,10 @@ def evaluateStaticGraphReconstruction(digraph, graph_embedding, estimated_adj = graph_embedding.get_reconstructed_adj(X_stat, node_l) else: estimated_adj = graph_embedding.get_reconstructed_adj(X_stat, file_suffix, node_l) - + predicted_edge_list = evaluation_util.getEdgeListFromAdjMtx(estimated_adj, is_undirected=is_undirected, edge_pairs=eval_edge_pairs) MAP = metrics.computeMAP(predicted_edge_list, digraph) prec_curv, _ = metrics.computePrecisionCurve(predicted_edge_list, digraph) - return (MAP, prec_curv) \ No newline at end of file + return (MAP, prec_curv) diff --git a/gem/utils/graph_util.py b/gem/utils/graph_util.py index 0861929..3cf3893 100644 --- a/gem/utils/graph_util.py +++ b/gem/utils/graph_util.py @@ -107,8 +107,8 @@ def saveGraphToEdgeListTxtn2v(graph, file_name): def loadGraphFromEdgeListTxt(file_name, directed=True): with open(file_name, 'r') as f: - n_nodes = f.readline() - f.readline() # Discard the number of edges + #n_nodes = f.readline() + #f.readline() # Discard the number of edges if directed: G = nx.DiGraph() else: diff --git a/test.py b/test.py index ee94336..6e5998e 100644 --- a/test.py +++ b/test.py @@ -1,10 +1,9 @@ import matplotlib.pyplot as plt +from time import time -from gem.utils import graph_util, plot_util +from gem.utils import graph_util, plot_util from gem.evaluation import visualize_embedding as viz from gem.evaluation import evaluate_graph_reconstruction as gr -from time import time - from gem.embedding.gf import GraphFactorization from gem.embedding.hope import HOPE @@ -16,7 +15,7 @@ # File that contains the edges. Format: source target # Optionally, you can add weights as third column: source target weight -edge_f = 'gem/data/karate.edgelist' +edge_f = 'gem/data/TEST_50M.edgelist' # Specify whether the edges are directed isDirected = True @@ -26,7 +25,7 @@ models = [] # You can comment out the methods you don't want to run -models.append(GraphFactorization(2, 100000, 1*10**-4, 1.0)) +models.append(GraphFactorization(2, 50000, 1*10**-4, 1.0)) models.append(HOPE(4, 0.01)) models.append(LaplacianEigenmaps(2)) models.append(LocallyLinearEmbedding(2)) @@ -36,7 +35,6 @@ weightfile=['./intermediate/enc_weights.hdf5', './intermediate/dec_weights.hdf5'])) - for embedding in models: print ('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) t1 = time() @@ -45,10 +43,9 @@ print (embedding._method_name+':\n\tTraining time: %f' % (time() - t1)) # Evaluate on graph reconstruction MAP, prec_curv = gr.evaluateStaticGraphReconstruction(G, embedding, Y, None) + #--------------------------------------------------------------------------------- + print(("\tMAP: {} \t preccision curve: {}\n\n\n\n"+'-'*100).format(MAP,prec_curv)) + #--------------------------------------------------------------------------------- # Visualize viz.plot_embedding2D(embedding.get_embedding(), di_graph=G, node_colors=None) plt.show() - - - -