From 1847336bd272121c833ef6e234775a57f73579d9 Mon Sep 17 00:00:00 2001 From: Ivan Butygin Date: Fri, 21 Feb 2020 14:51:36 +0300 Subject: [PATCH 1/4] tbb concurrent hash example --- sdc/_concurrent_hash.cpp | 137 ++++++++++++++++++++++++++++++++++++ sdc/concurrent_hash.py | 116 ++++++++++++++++++++++++++++++ sdc/tests/test_dataframe.py | 27 +++++++ setup.py | 54 ++++++++++++++ 4 files changed, 334 insertions(+) create mode 100644 sdc/_concurrent_hash.cpp create mode 100644 sdc/concurrent_hash.py diff --git a/sdc/_concurrent_hash.cpp b/sdc/_concurrent_hash.cpp new file mode 100644 index 000000000..c8c7c040d --- /dev/null +++ b/sdc/_concurrent_hash.cpp @@ -0,0 +1,137 @@ +//***************************************************************************** +// Copyright (c) 2020, Intel Corporation All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************** + +#include +#include +#include +#include + + +template +using hashmap = tbb::concurrent_hash_map>; + +template +using iter_range = std::pair::iterator, typename hashmap::iterator>; + +using int_hashmap = hashmap; +using int_hashmap_iters = iter_range; + +extern "C" +{ +void* create_int_hashmap() +{ + return new int_hashmap; +} + +void delete_int_hashmap(void* obj) +{ + delete static_cast(obj); +} + +void addelem_int_hashmap(void* obj, int64_t key, size_t val) +{ + auto& h = *static_cast(obj); + int_hashmap::accessor ac; + h.insert(ac, key); + auto& vec = ac->second; + ac.release(); + vec.push_back(val); + // h[key].push_back(val); +} + +void* createiter_int_hashmap(void* obj) +{ + auto& h = *static_cast(obj); + return new int_hashmap_iters{h.begin(), h.end()}; +} + +int32_t enditer_int_hashmap(void* it) +{ + auto& r = *static_cast(it); + return static_cast(r.first == r.second); +} + +void nextiter_int_hashmap(void* it) +{ + auto& r = *static_cast(it); + ++r.first; +} + +int64_t iterkey_int_hashmap(void* it) +{ + auto& r = *static_cast(it); + return r.first->first; +} + +size_t itersize_int_hashmap(void* it) +{ + auto& r = *static_cast(it); + return r.first->second.size(); +} + +size_t iterelem_int_hashmap(void* it, size_t index) +{ + auto& r = *static_cast(it); + return r.first->second[index]; +} + +void deleteiter_int_hashmap(void* obj) +{ + delete static_cast(obj); +} + + +PyMODINIT_FUNC PyInit_hconcurrent_hash() +{ + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "hconcurrent_hash", + "No docs", + -1, + NULL, + }; + PyObject* m = PyModule_Create(&moduledef); + if (m == NULL) + { + return NULL; + } + +#define REGISTER(func) PyObject_SetAttrString(m, #func, PyLong_FromVoidPtr((void*)(&func))); + REGISTER(create_int_hashmap) + REGISTER(delete_int_hashmap) + REGISTER(addelem_int_hashmap) + + REGISTER(createiter_int_hashmap) + REGISTER(enditer_int_hashmap) + REGISTER(nextiter_int_hashmap) + REGISTER(iterkey_int_hashmap) + REGISTER(itersize_int_hashmap) + REGISTER(iterelem_int_hashmap) + REGISTER(deleteiter_int_hashmap) +#undef REGISTER + return m; +} +} diff --git a/sdc/concurrent_hash.py b/sdc/concurrent_hash.py new file mode 100644 index 000000000..514a19a68 --- /dev/null +++ b/sdc/concurrent_hash.py @@ -0,0 +1,116 @@ +import numba +import sdc + +from numba import types, typing, generated_jit +from numba.extending import models, register_model +from numba.extending import lower_builtin, overload_method, overload, intrinsic + +from llvmlite import ir as lir +import llvmlite.binding as ll +from . import hconcurrent_hash +ll.add_symbol('create_int_hashmap', hconcurrent_hash.create_int_hashmap) +ll.add_symbol('delete_int_hashmap', hconcurrent_hash.delete_int_hashmap) +ll.add_symbol('addelem_int_hashmap',hconcurrent_hash.addelem_int_hashmap) + +ll.add_symbol('createiter_int_hashmap',hconcurrent_hash.createiter_int_hashmap) +ll.add_symbol('enditer_int_hashmap',hconcurrent_hash.enditer_int_hashmap) +ll.add_symbol('nextiter_int_hashmap',hconcurrent_hash.nextiter_int_hashmap) +ll.add_symbol('iterkey_int_hashmap',hconcurrent_hash.iterkey_int_hashmap) +ll.add_symbol('itersize_int_hashmap',hconcurrent_hash.itersize_int_hashmap) +ll.add_symbol('iterelem_int_hashmap',hconcurrent_hash.iterelem_int_hashmap) +ll.add_symbol('deleteiter_int_hashmap',hconcurrent_hash.deleteiter_int_hashmap) + +_create_int_hashmap = types.ExternalFunction("create_int_hashmap", + types.voidptr()) +_delete_int_hashmap = types.ExternalFunction("delete_int_hashmap", + types.void(types.voidptr)) +_addelem_int_hashmap = types.ExternalFunction("addelem_int_hashmap", + types.void(types.voidptr, types.int64, types.intp)) + +_createiter_int_hashmap = types.ExternalFunction("createiter_int_hashmap", + types.voidptr(types.voidptr)) +_enditer_int_hashmap = types.ExternalFunction("enditer_int_hashmap", + types.int32(types.voidptr)) +_nextiter_int_hashmap = types.ExternalFunction("nextiter_int_hashmap", + types.void(types.voidptr)) +_iterkey_int_hashmap = types.ExternalFunction("iterkey_int_hashmap", + types.int64(types.voidptr)) +_itersize_int_hashmap = types.ExternalFunction("itersize_int_hashmap", + types.intp(types.voidptr)) +_iterelem_int_hashmap = types.ExternalFunction("iterelem_int_hashmap", + types.intp(types.voidptr, types.intp)) +_deleteiter_int_hashmap = types.ExternalFunction("deleteiter_int_hashmap", + types.void(types.voidptr)) + + +def create_int_hashmap(): + pass + +def delete_int_hashmap(): + pass + +def addelem_int_hashmap(): + pass + + +def createiter_int_hashmap(): + pass + +def enditer_int_hashmap(): + pass + +def nextiter_int_hashmap(): + pass + +def iterkey_int_hashmap(): + pass + +def itersize_int_hashmap(): + pass + +def iterelem_int_hashmap(): + pass + +def deleteiter_int_hashmap(): + pass + +@overload(create_int_hashmap) +def create_int_hashmap_overload(): + return lambda: _create_int_hashmap() + +@overload(delete_int_hashmap) +def delete_int_hashmap_overload(h): + return lambda h: _delete_int_hashmap(h) + +@overload(addelem_int_hashmap) +def addelem_int_hashmap_overload(h, key, val): + return lambda h, key, val: _addelem_int_hashmap(h, key, val) + + +@overload(createiter_int_hashmap) +def createiter_int_hashmap_overload(h): + return lambda h: _createiter_int_hashmap(h) + +@overload(enditer_int_hashmap) +def enditer_int_hashmap_overload(h): + return lambda h: _enditer_int_hashmap(h) + +@overload(nextiter_int_hashmap) +def nextiter_int_hashmap_overload(h): + return lambda h: _nextiter_int_hashmap(h) + +@overload(iterkey_int_hashmap) +def iterkey_int_hashmap_overload(h): + return lambda h: _iterkey_int_hashmap(h) + +@overload(itersize_int_hashmap) +def itersize_int_hashmap_overload(h): + return lambda h: _itersize_int_hashmap(h) + +@overload(iterelem_int_hashmap) +def iterelem_int_hashmap_overload(h, i): + return lambda h, i: _iterelem_int_hashmap(h, i) + +@overload(deleteiter_int_hashmap) +def deleteiter_int_hashmap_overload(h): + return lambda h: _deleteiter_int_hashmap(h) diff --git a/sdc/tests/test_dataframe.py b/sdc/tests/test_dataframe.py index 5008e8d4d..a5cc2e9f8 100644 --- a/sdc/tests/test_dataframe.py +++ b/sdc/tests/test_dataframe.py @@ -1791,6 +1791,33 @@ def test_impl(): pd.testing.assert_series_equal(hpat_func(), test_impl()) + def test_tbb(self): + import sdc.concurrent_hash + def test_impl(): + h = sdc.concurrent_hash.create_int_hashmap() + + sdc.concurrent_hash.addelem_int_hashmap(h,1,2) + sdc.concurrent_hash.addelem_int_hashmap(h,1,3) + sdc.concurrent_hash.addelem_int_hashmap(h,1,4) + sdc.concurrent_hash.addelem_int_hashmap(h,1,5) + sdc.concurrent_hash.addelem_int_hashmap(h,2,6) + + it = sdc.concurrent_hash.createiter_int_hashmap(h) + while 0 == sdc.concurrent_hash.enditer_int_hashmap(it): + key = sdc.concurrent_hash.iterkey_int_hashmap(it) + sz = sdc.concurrent_hash.itersize_int_hashmap(it) + for i in range(sz): + val = sdc.concurrent_hash.iterelem_int_hashmap(it, i) + print(key, val) + + sdc.concurrent_hash.nextiter_int_hashmap(it) + + sdc.concurrent_hash.deleteiter_int_hashmap(it) + sdc.concurrent_hash.delete_int_hashmap(h) + + hpat_func = self.jit(test_impl) + hpat_func() + if __name__ == "__main__": unittest.main() diff --git a/setup.py b/setup.py index f9b08004e..0e8c0e0dc 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ # ***************************************************************************** from setuptools import setup, Extension, find_packages, Command +import sys import platform import os from docs.source.buildscripts.sdc_build_doc import SDCBuildDoc @@ -197,6 +198,59 @@ def readme(): if _has_opencv: _ext_mods.append(ext_cv_wrapper) +# Copypaste from numba +def check_file_at_path(path2file): + """ + Takes a list as a path, a single glob (*) is permitted as an entry which + indicates that expansion at this location is required (i.e. version + might not be known). + """ + found = None + path2check = [os.path.split(os.path.split(sys.executable)[0])[0]] + path2check += [os.getenv(n, '') for n in ['CONDA_PREFIX', 'PREFIX']] + if sys.platform.startswith('win'): + path2check += [os.path.join(p, 'Library') for p in path2check] + for p in path2check: + if p: + if '*' in path2file: + globloc = path2file.index('*') + searchroot = os.path.join(*path2file[:globloc]) + try: + potential_locs = os.listdir(os.path.join(p, searchroot)) + except BaseException: + continue + searchfor = path2file[globloc + 1:] + for x in potential_locs: + potpath = os.path.join(p, searchroot, x, *searchfor) + if os.path.isfile(potpath): + found = p # the latest is used + elif os.path.isfile(os.path.join(p, *path2file)): + found = p # the latest is used + return found + +# Search for Intel TBB, first check env var TBBROOT then conda locations +tbb_root = os.getenv('TBBROOT') +if not tbb_root: + tbb_root = check_file_at_path(['include', 'tbb', 'tbb.h']) + +print("Using Intel TBB from:", tbb_root) +ext_hconcurrent_hash = Extension( + name="sdc.hconcurrent_hash", + sources=["sdc/_concurrent_hash.cpp"], + include_dirs=[os.path.join(tbb_root, 'include')], + libraries=['tbb'], + library_dirs=[ + # for Linux + os.path.join(tbb_root, 'lib', 'intel64', 'gcc4.4'), + # for MacOS + os.path.join(tbb_root, 'lib'), + # for Windows + os.path.join(tbb_root, 'lib', 'intel64', 'vc_mt'), + ], + language="c++", + ) + +_ext_mods.append(ext_hconcurrent_hash) class style(Command): """ Command to check and adjust code style From 213eccc62c382391508374a1b1730d94493950d4 Mon Sep 17 00:00:00 2001 From: Ivan Butygin Date: Fri, 21 Feb 2020 15:17:44 +0300 Subject: [PATCH 2/4] style --- sdc/concurrent_hash.py | 59 ++++++++++++++++++++++++++++++++----- sdc/tests/test_dataframe.py | 11 +++---- setup.py | 56 ++++++++++++++++++----------------- 3 files changed, 86 insertions(+), 40 deletions(-) diff --git a/sdc/concurrent_hash.py b/sdc/concurrent_hash.py index 514a19a68..000ccbda5 100644 --- a/sdc/concurrent_hash.py +++ b/sdc/concurrent_hash.py @@ -1,3 +1,29 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + import numba import sdc @@ -10,15 +36,15 @@ from . import hconcurrent_hash ll.add_symbol('create_int_hashmap', hconcurrent_hash.create_int_hashmap) ll.add_symbol('delete_int_hashmap', hconcurrent_hash.delete_int_hashmap) -ll.add_symbol('addelem_int_hashmap',hconcurrent_hash.addelem_int_hashmap) +ll.add_symbol('addelem_int_hashmap', hconcurrent_hash.addelem_int_hashmap) -ll.add_symbol('createiter_int_hashmap',hconcurrent_hash.createiter_int_hashmap) -ll.add_symbol('enditer_int_hashmap',hconcurrent_hash.enditer_int_hashmap) -ll.add_symbol('nextiter_int_hashmap',hconcurrent_hash.nextiter_int_hashmap) -ll.add_symbol('iterkey_int_hashmap',hconcurrent_hash.iterkey_int_hashmap) -ll.add_symbol('itersize_int_hashmap',hconcurrent_hash.itersize_int_hashmap) -ll.add_symbol('iterelem_int_hashmap',hconcurrent_hash.iterelem_int_hashmap) -ll.add_symbol('deleteiter_int_hashmap',hconcurrent_hash.deleteiter_int_hashmap) +ll.add_symbol('createiter_int_hashmap', hconcurrent_hash.createiter_int_hashmap) +ll.add_symbol('enditer_int_hashmap', hconcurrent_hash.enditer_int_hashmap) +ll.add_symbol('nextiter_int_hashmap', hconcurrent_hash.nextiter_int_hashmap) +ll.add_symbol('iterkey_int_hashmap', hconcurrent_hash.iterkey_int_hashmap) +ll.add_symbol('itersize_int_hashmap', hconcurrent_hash.itersize_int_hashmap) +ll.add_symbol('iterelem_int_hashmap', hconcurrent_hash.iterelem_int_hashmap) +ll.add_symbol('deleteiter_int_hashmap', hconcurrent_hash.deleteiter_int_hashmap) _create_int_hashmap = types.ExternalFunction("create_int_hashmap", types.voidptr()) @@ -46,9 +72,11 @@ def create_int_hashmap(): pass + def delete_int_hashmap(): pass + def addelem_int_hashmap(): pass @@ -56,32 +84,41 @@ def addelem_int_hashmap(): def createiter_int_hashmap(): pass + def enditer_int_hashmap(): pass + def nextiter_int_hashmap(): pass + def iterkey_int_hashmap(): pass + def itersize_int_hashmap(): pass + def iterelem_int_hashmap(): pass + def deleteiter_int_hashmap(): pass + @overload(create_int_hashmap) def create_int_hashmap_overload(): return lambda: _create_int_hashmap() + @overload(delete_int_hashmap) def delete_int_hashmap_overload(h): return lambda h: _delete_int_hashmap(h) + @overload(addelem_int_hashmap) def addelem_int_hashmap_overload(h, key, val): return lambda h, key, val: _addelem_int_hashmap(h, key, val) @@ -91,26 +128,32 @@ def addelem_int_hashmap_overload(h, key, val): def createiter_int_hashmap_overload(h): return lambda h: _createiter_int_hashmap(h) + @overload(enditer_int_hashmap) def enditer_int_hashmap_overload(h): return lambda h: _enditer_int_hashmap(h) + @overload(nextiter_int_hashmap) def nextiter_int_hashmap_overload(h): return lambda h: _nextiter_int_hashmap(h) + @overload(iterkey_int_hashmap) def iterkey_int_hashmap_overload(h): return lambda h: _iterkey_int_hashmap(h) + @overload(itersize_int_hashmap) def itersize_int_hashmap_overload(h): return lambda h: _itersize_int_hashmap(h) + @overload(iterelem_int_hashmap) def iterelem_int_hashmap_overload(h, i): return lambda h, i: _iterelem_int_hashmap(h, i) + @overload(deleteiter_int_hashmap) def deleteiter_int_hashmap_overload(h): return lambda h: _deleteiter_int_hashmap(h) diff --git a/sdc/tests/test_dataframe.py b/sdc/tests/test_dataframe.py index a5cc2e9f8..4aac01b8f 100644 --- a/sdc/tests/test_dataframe.py +++ b/sdc/tests/test_dataframe.py @@ -1793,14 +1793,15 @@ def test_impl(): def test_tbb(self): import sdc.concurrent_hash + def test_impl(): h = sdc.concurrent_hash.create_int_hashmap() - sdc.concurrent_hash.addelem_int_hashmap(h,1,2) - sdc.concurrent_hash.addelem_int_hashmap(h,1,3) - sdc.concurrent_hash.addelem_int_hashmap(h,1,4) - sdc.concurrent_hash.addelem_int_hashmap(h,1,5) - sdc.concurrent_hash.addelem_int_hashmap(h,2,6) + sdc.concurrent_hash.addelem_int_hashmap(h, 1, 2) + sdc.concurrent_hash.addelem_int_hashmap(h, 1, 3) + sdc.concurrent_hash.addelem_int_hashmap(h, 1, 4) + sdc.concurrent_hash.addelem_int_hashmap(h, 1, 5) + sdc.concurrent_hash.addelem_int_hashmap(h, 2, 6) it = sdc.concurrent_hash.createiter_int_hashmap(h) while 0 == sdc.concurrent_hash.enditer_int_hashmap(it): diff --git a/setup.py b/setup.py index 0e8c0e0dc..e1ce97c33 100644 --- a/setup.py +++ b/setup.py @@ -198,35 +198,37 @@ def readme(): if _has_opencv: _ext_mods.append(ext_cv_wrapper) + # Copypaste from numba def check_file_at_path(path2file): - """ - Takes a list as a path, a single glob (*) is permitted as an entry which - indicates that expansion at this location is required (i.e. version - might not be known). - """ - found = None - path2check = [os.path.split(os.path.split(sys.executable)[0])[0]] - path2check += [os.getenv(n, '') for n in ['CONDA_PREFIX', 'PREFIX']] - if sys.platform.startswith('win'): - path2check += [os.path.join(p, 'Library') for p in path2check] - for p in path2check: - if p: - if '*' in path2file: - globloc = path2file.index('*') - searchroot = os.path.join(*path2file[:globloc]) - try: - potential_locs = os.listdir(os.path.join(p, searchroot)) - except BaseException: - continue - searchfor = path2file[globloc + 1:] - for x in potential_locs: - potpath = os.path.join(p, searchroot, x, *searchfor) - if os.path.isfile(potpath): - found = p # the latest is used - elif os.path.isfile(os.path.join(p, *path2file)): - found = p # the latest is used - return found + """ + Takes a list as a path, a single glob (*) is permitted as an entry which + indicates that expansion at this location is required (i.e. version + might not be known). + """ + found = None + path2check = [os.path.split(os.path.split(sys.executable)[0])[0]] + path2check += [os.getenv(n, '') for n in ['CONDA_PREFIX', 'PREFIX']] + if sys.platform.startswith('win'): + path2check += [os.path.join(p, 'Library') for p in path2check] + for p in path2check: + if p: + if '*' in path2file: + globloc = path2file.index('*') + searchroot = os.path.join(*path2file[:globloc]) + try: + potential_locs = os.listdir(os.path.join(p, searchroot)) + except BaseException: + continue + searchfor = path2file[globloc + 1:] + for x in potential_locs: + potpath = os.path.join(p, searchroot, x, *searchfor) + if os.path.isfile(potpath): + found = p # the latest is used + elif os.path.isfile(os.path.join(p, *path2file)): + found = p # the latest is used + return found + # Search for Intel TBB, first check env var TBBROOT then conda locations tbb_root = os.getenv('TBBROOT') From c59fd1f6b21de996eb3359ffa7d622c5295c2f47 Mon Sep 17 00:00:00 2001 From: Ivan Butygin Date: Fri, 21 Feb 2020 15:37:27 +0300 Subject: [PATCH 3/4] multimap --- sdc/_concurrent_hash.cpp | 25 ++++++------------------- sdc/concurrent_hash.py | 26 +++++++------------------- sdc/tests/test_dataframe.py | 6 ++---- 3 files changed, 15 insertions(+), 42 deletions(-) diff --git a/sdc/_concurrent_hash.cpp b/sdc/_concurrent_hash.cpp index c8c7c040d..3bfda0036 100644 --- a/sdc/_concurrent_hash.cpp +++ b/sdc/_concurrent_hash.cpp @@ -26,12 +26,11 @@ #include #include -#include -#include +#include template -using hashmap = tbb::concurrent_hash_map>; +using hashmap = tbb::concurrent_unordered_multimap; template using iter_range = std::pair::iterator, typename hashmap::iterator>; @@ -54,12 +53,7 @@ void delete_int_hashmap(void* obj) void addelem_int_hashmap(void* obj, int64_t key, size_t val) { auto& h = *static_cast(obj); - int_hashmap::accessor ac; - h.insert(ac, key); - auto& vec = ac->second; - ac.release(); - vec.push_back(val); - // h[key].push_back(val); + h.insert({key,val}); } void* createiter_int_hashmap(void* obj) @@ -86,16 +80,10 @@ int64_t iterkey_int_hashmap(void* it) return r.first->first; } -size_t itersize_int_hashmap(void* it) +size_t iterval_int_hashmap(void* it) { auto& r = *static_cast(it); - return r.first->second.size(); -} - -size_t iterelem_int_hashmap(void* it, size_t index) -{ - auto& r = *static_cast(it); - return r.first->second[index]; + return r.first->second; } void deleteiter_int_hashmap(void* obj) @@ -128,8 +116,7 @@ PyMODINIT_FUNC PyInit_hconcurrent_hash() REGISTER(enditer_int_hashmap) REGISTER(nextiter_int_hashmap) REGISTER(iterkey_int_hashmap) - REGISTER(itersize_int_hashmap) - REGISTER(iterelem_int_hashmap) + REGISTER(iterval_int_hashmap) REGISTER(deleteiter_int_hashmap) #undef REGISTER return m; diff --git a/sdc/concurrent_hash.py b/sdc/concurrent_hash.py index 000ccbda5..745b5eab1 100644 --- a/sdc/concurrent_hash.py +++ b/sdc/concurrent_hash.py @@ -42,8 +42,7 @@ ll.add_symbol('enditer_int_hashmap', hconcurrent_hash.enditer_int_hashmap) ll.add_symbol('nextiter_int_hashmap', hconcurrent_hash.nextiter_int_hashmap) ll.add_symbol('iterkey_int_hashmap', hconcurrent_hash.iterkey_int_hashmap) -ll.add_symbol('itersize_int_hashmap', hconcurrent_hash.itersize_int_hashmap) -ll.add_symbol('iterelem_int_hashmap', hconcurrent_hash.iterelem_int_hashmap) +ll.add_symbol('iterval_int_hashmap', hconcurrent_hash.iterval_int_hashmap) ll.add_symbol('deleteiter_int_hashmap', hconcurrent_hash.deleteiter_int_hashmap) _create_int_hashmap = types.ExternalFunction("create_int_hashmap", @@ -61,10 +60,8 @@ types.void(types.voidptr)) _iterkey_int_hashmap = types.ExternalFunction("iterkey_int_hashmap", types.int64(types.voidptr)) -_itersize_int_hashmap = types.ExternalFunction("itersize_int_hashmap", - types.intp(types.voidptr)) -_iterelem_int_hashmap = types.ExternalFunction("iterelem_int_hashmap", - types.intp(types.voidptr, types.intp)) +_iterval_int_hashmap = types.ExternalFunction("iterval_int_hashmap", + types.intp(types.voidptr)) _deleteiter_int_hashmap = types.ExternalFunction("deleteiter_int_hashmap", types.void(types.voidptr)) @@ -97,11 +94,7 @@ def iterkey_int_hashmap(): pass -def itersize_int_hashmap(): - pass - - -def iterelem_int_hashmap(): +def iterval_int_hashmap(): pass @@ -144,14 +137,9 @@ def iterkey_int_hashmap_overload(h): return lambda h: _iterkey_int_hashmap(h) -@overload(itersize_int_hashmap) -def itersize_int_hashmap_overload(h): - return lambda h: _itersize_int_hashmap(h) - - -@overload(iterelem_int_hashmap) -def iterelem_int_hashmap_overload(h, i): - return lambda h, i: _iterelem_int_hashmap(h, i) +@overload(iterval_int_hashmap) +def iterval_int_hashmap_overload(h): + return lambda h: _iterval_int_hashmap(h) @overload(deleteiter_int_hashmap) diff --git a/sdc/tests/test_dataframe.py b/sdc/tests/test_dataframe.py index 4aac01b8f..2946e2444 100644 --- a/sdc/tests/test_dataframe.py +++ b/sdc/tests/test_dataframe.py @@ -1806,10 +1806,8 @@ def test_impl(): it = sdc.concurrent_hash.createiter_int_hashmap(h) while 0 == sdc.concurrent_hash.enditer_int_hashmap(it): key = sdc.concurrent_hash.iterkey_int_hashmap(it) - sz = sdc.concurrent_hash.itersize_int_hashmap(it) - for i in range(sz): - val = sdc.concurrent_hash.iterelem_int_hashmap(it, i) - print(key, val) + val = sdc.concurrent_hash.iterval_int_hashmap(it) + print(key, val) sdc.concurrent_hash.nextiter_int_hashmap(it) From 08f9c1effa2399a70522fc29fc7d5c604bf7e3f9 Mon Sep 17 00:00:00 2001 From: Ivan Butygin Date: Fri, 28 Feb 2020 16:20:01 +0300 Subject: [PATCH 4/4] Function pointer example --- sdc/_concurrent_hash.cpp | 12 ++++++++++++ sdc/concurrent_hash.py | 27 +++++++++++++++++++++++++-- sdc/tests/test_dataframe.py | 20 +++++++++++++++++--- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/sdc/_concurrent_hash.cpp b/sdc/_concurrent_hash.cpp index 3bfda0036..b76075caa 100644 --- a/sdc/_concurrent_hash.cpp +++ b/sdc/_concurrent_hash.cpp @@ -91,6 +91,16 @@ void deleteiter_int_hashmap(void* obj) delete static_cast(obj); } +using funcptr_t = int32_t(*)(int32_t,int32_t,int32_t); +int32_t test_funcptr(funcptr_t func, int32_t a, int32_t b) +{ + int32_t res = 0; + for (int i = 0; i < 10; ++i) + { + res += func(a, b, i); + } + return res; +} PyMODINIT_FUNC PyInit_hconcurrent_hash() { @@ -118,6 +128,8 @@ PyMODINIT_FUNC PyInit_hconcurrent_hash() REGISTER(iterkey_int_hashmap) REGISTER(iterval_int_hashmap) REGISTER(deleteiter_int_hashmap) + + REGISTER(test_funcptr) #undef REGISTER return m; } diff --git a/sdc/concurrent_hash.py b/sdc/concurrent_hash.py index 745b5eab1..ecdac6c1f 100644 --- a/sdc/concurrent_hash.py +++ b/sdc/concurrent_hash.py @@ -28,8 +28,7 @@ import sdc from numba import types, typing, generated_jit -from numba.extending import models, register_model -from numba.extending import lower_builtin, overload_method, overload, intrinsic +from numba.extending import lower_builtin, overload_method, overload, intrinsic, register_jitable from llvmlite import ir as lir import llvmlite.binding as ll @@ -45,6 +44,8 @@ ll.add_symbol('iterval_int_hashmap', hconcurrent_hash.iterval_int_hashmap) ll.add_symbol('deleteiter_int_hashmap', hconcurrent_hash.deleteiter_int_hashmap) +ll.add_symbol('test_funcptr', hconcurrent_hash.test_funcptr) + _create_int_hashmap = types.ExternalFunction("create_int_hashmap", types.voidptr()) _delete_int_hashmap = types.ExternalFunction("delete_int_hashmap", @@ -65,6 +66,9 @@ _deleteiter_int_hashmap = types.ExternalFunction("deleteiter_int_hashmap", types.void(types.voidptr)) +_test_funcptr = types.ExternalFunction("test_funcptr", + types.int32(types.voidptr,types.int32,types.int32)) + def create_int_hashmap(): pass @@ -102,6 +106,10 @@ def deleteiter_int_hashmap(): pass +def test_funcptr(): + pass + + @overload(create_int_hashmap) def create_int_hashmap_overload(): return lambda: _create_int_hashmap() @@ -145,3 +153,18 @@ def iterval_int_hashmap_overload(h): @overload(deleteiter_int_hashmap) def deleteiter_int_hashmap_overload(h): return lambda h: _deleteiter_int_hashmap(h) + + + +@register_jitable +def sink(*args): + args[0] + +@overload(test_funcptr) +def test_funcptr_overload(a,b,c): + def func(a,b,c): + res = _test_funcptr(a,b,c) + sink(a,b,c) + return res + + return func diff --git a/sdc/tests/test_dataframe.py b/sdc/tests/test_dataframe.py index 2946e2444..59661dcda 100644 --- a/sdc/tests/test_dataframe.py +++ b/sdc/tests/test_dataframe.py @@ -1794,7 +1794,14 @@ def test_impl(): def test_tbb(self): import sdc.concurrent_hash - def test_impl(): + @numba.cfunc("int32(int32, int32, int32)") + def callback(x, y, z): + return x + y + z + + global funcptr + funcptr = callback.address + + def test_impl1(): h = sdc.concurrent_hash.create_int_hashmap() sdc.concurrent_hash.addelem_int_hashmap(h, 1, 2) @@ -1814,8 +1821,15 @@ def test_impl(): sdc.concurrent_hash.deleteiter_int_hashmap(it) sdc.concurrent_hash.delete_int_hashmap(h) - hpat_func = self.jit(test_impl) - hpat_func() + hpat_func1 = self.jit(test_impl1) + hpat_func1() + + def test_impl2(): + r = sdc.concurrent_hash.test_funcptr(funcptr, 2, 3) + print('res', r) + + hpat_func2 = self.jit(test_impl2) + hpat_func2() if __name__ == "__main__":