Skip to content

Commit d055972

Browse files
texzonetexzone
authored
Test (#2)
* update * update * update * update * update * update * update * update * try again * try again * update * update * update * update * update * update * update * update * update * update * update * update * update * update
1 parent 571e123 commit d055972

File tree

827 files changed

+251061
-6
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

827 files changed

+251061
-6
lines changed

‎.github/workflows/build_wheels.yml‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,27 @@ name: Build
22

33
on: [push, pull_request]
44

5+
56
jobs:
67
build_wheels:
7-
name: Build wheels on ${{ matrix.os }}
8-
runs-on: ${{ matrix.os }}
9-
strategy:
10-
matrix:
11-
os: [ubuntu-20.04, windows-2019, macos-10.15]
8+
name: Build wheels on ubuntu-20.04
9+
runs-on: ubuntu-20.04
10+
# strategy:
11+
# matrix:
12+
# os: [ubuntu-20.04, windows-2019, macos-10.15]
1213

1314
steps:
15+
# - run: |
16+
# sudo apt install graphviz
17+
# wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.8.1.tar.gz
18+
# tar -xzvf openfst-1.8.1.tar.gz
19+
# cd openfst-1.8.1/
20+
# ./configure --enable-far=true
21+
# make -j4
22+
# sudo make install
23+
# echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib" >> ~/.bashrc
24+
# source ~/.bashrc
25+
1426
- uses: actions/checkout@v2
1527

1628
- name: Build wheels

‎ThreadPool.h‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#ifndef THREAD_POOL_H
2+
#define THREAD_POOL_H
3+
4+
#include <vector>
5+
#include <queue>
6+
#include <memory>
7+
#include <thread>
8+
#include <mutex>
9+
#include <condition_variable>
10+
#include <future>
11+
#include <functional>
12+
#include <stdexcept>
13+
14+
class ThreadPool {
15+
public:
16+
ThreadPool(size_t);
17+
template<class F, class... Args>
18+
auto enqueue(F&& f, Args&&... args)
19+
-> std::future<typename std::result_of<F(Args...)>::type>;
20+
~ThreadPool();
21+
private:
22+
// need to keep track of threads so we can join them
23+
std::vector< std::thread > workers;
24+
// the task queue
25+
std::queue< std::function<void()> > tasks;
26+
27+
// synchronization
28+
std::mutex queue_mutex;
29+
std::condition_variable condition;
30+
bool stop;
31+
};
32+
33+
// the constructor just launches some amount of workers
34+
inline ThreadPool::ThreadPool(size_t threads)
35+
: stop(false)
36+
{
37+
for(size_t i = 0;i<threads;++i)
38+
workers.emplace_back(
39+
[this]
40+
{
41+
for(;;)
42+
{
43+
std::function<void()> task;
44+
45+
{
46+
std::unique_lock<std::mutex> lock(this->queue_mutex);
47+
this->condition.wait(lock,
48+
[this]{ return this->stop || !this->tasks.empty(); });
49+
if(this->stop && this->tasks.empty())
50+
return;
51+
task = std::move(this->tasks.front());
52+
this->tasks.pop();
53+
}
54+
55+
task();
56+
}
57+
}
58+
);
59+
}
60+
61+
// add new work item to the pool
62+
template<class F, class... Args>
63+
auto ThreadPool::enqueue(F&& f, Args&&... args)
64+
-> std::future<typename std::result_of<F(Args...)>::type>
65+
{
66+
using return_type = typename std::result_of<F(Args...)>::type;
67+
68+
auto task = std::make_shared< std::packaged_task<return_type()> >(
69+
std::bind(std::forward<F>(f), std::forward<Args>(args)...)
70+
);
71+
72+
std::future<return_type> res = task->get_future();
73+
{
74+
std::unique_lock<std::mutex> lock(queue_mutex);
75+
76+
// don't allow enqueueing after stopping the pool
77+
if(stop)
78+
throw std::runtime_error("enqueue on stopped ThreadPool");
79+
80+
tasks.emplace([task](){ (*task)(); });
81+
}
82+
condition.notify_one();
83+
return res;
84+
}
85+
86+
// the destructor joins all threads
87+
inline ThreadPool::~ThreadPool()
88+
{
89+
{
90+
std::unique_lock<std::mutex> lock(queue_mutex);
91+
stop = true;
92+
}
93+
condition.notify_all();
94+
for(std::thread &worker: workers)
95+
worker.join();
96+
}
97+
98+
#endif

‎_init_paths.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ def add_path(path):
1414

1515
this_dir = os.path.dirname(__file__)
1616

17+
1718
# Add project path to PYTHONPATH
1819
proj_path = os.path.join(this_dir, '..')
1920
add_path(proj_path)

‎kenlm/.github/workflows/mac.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Mac
2+
3+
on:
4+
push:
5+
branches: master
6+
pull_request:
7+
branches: master
8+
9+
jobs:
10+
build:
11+
runs-on: macOS-latest
12+
13+
steps:
14+
- uses: actions/checkout@v2
15+
- name: Install Boost
16+
run: |
17+
brew install boost
18+
brew install libomp
19+
brew install eigen
20+
- name: cmake
21+
run: |
22+
cmake -E make_directory build
23+
cd build
24+
cmake ..
25+
- name: Compile
26+
working-directory: build
27+
run: cmake --build . -j2
28+
- name: Test
29+
working-directory: build
30+
run: ctest -j2

‎kenlm/.github/workflows/ubuntu.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Ubuntu
2+
3+
on:
4+
push:
5+
branches: [master]
6+
pull_request:
7+
branches: [master]
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- uses: actions/checkout@v2
15+
- name: dependencies
16+
run: sudo apt-get install -y build-essential libboost-all-dev cmake zlib1g-dev libbz2-dev liblzma-dev
17+
- name: cmake
18+
run: |
19+
cmake -E make_directory build
20+
cd build
21+
cmake -DCOMPILE_TESTS=ON ..
22+
- name: Compile
23+
working-directory: build
24+
run: cmake --build . -j2
25+
- name: Test
26+
working-directory: build
27+
run: ctest -j2

‎kenlm/.github/workflows/windows.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: Windows
2+
3+
on:
4+
push:
5+
branches: [master]
6+
pull_request:
7+
branches: [master]
8+
9+
jobs:
10+
build:
11+
runs-on: windows-latest
12+
13+
steps:
14+
- uses: actions/checkout@v2
15+
- name: cmake
16+
run: |
17+
cmake -E make_directory build
18+
cd build
19+
cmake -DBOOST_ROOT="${env:BOOST_ROOT_1_72_0}" ..
20+
- name: Compile
21+
working-directory: build
22+
run: cmake --build . -j2
23+
- name: Test
24+
working-directory: build
25+
run: ctest -j2

‎kenlm/.gitignore‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
util/file_piece.cc.gz
2+
*.swp
3+
*.o
4+
doc/
5+
build/
6+
/bin
7+
/lib
8+
/tests
9+
._*
10+
windows/Win32
11+
windows/x64
12+
windows/*.user
13+
windows/*.sdf
14+
windows/*.opensdf
15+
windows/*.suo
16+
CMakeFiles
17+
cmake_install.cmake
18+
CMakeCache.txt
19+
CTestTestfile.cmake
20+
DartConfiguration.tcl
21+
Makefile

‎kenlm/BUILDING‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
KenLM has switched to cmake
2+
cmake .
3+
make -j 4
4+
But they recommend building out of tree
5+
mkdir -p build && cd build
6+
cmake ..
7+
make -j 4
8+
9+
If you only want the query code and do not care about compression (.gz, .bz2, and .xz):
10+
./compile_query_only.sh
11+
12+
Windows:
13+
The windows directory has visual studio files. Note that you need to compile
14+
the kenlm project before build_binary and ngram_query projects.
15+
16+
OSX:
17+
Missing dependencies can be remedied with brew.
18+
brew install cmake boost eigen
19+
20+
Debian/Ubuntu:
21+
sudo apt install build-essential cmake libboost-system-dev libboost-thread-dev libboost-program-options-dev libboost-test-dev libeigen3-dev zlib1g-dev libbz2-dev liblzma-dev

‎kenlm/CMakeLists.txt‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
cmake_minimum_required(VERSION 3.1)
2+
3+
if (WIN32)
4+
set(Boost_USE_STATIC_LIBS OFF)
5+
# The auto-linking feature has problems with USE_STATIC_LIBS off, so we use
6+
# BOOST_ALL_NO_LIB to turn it off.
7+
# Several boost libraries headers aren't configured correctly if
8+
# USE_STATIC_LIBS is off, so we explicitly say they are dynamic with the
9+
# remaining definitions.
10+
add_definitions(-DBOOST_ALL_NO_LIB -DBOOST_PROGRAM_OPTIONS_DYN_LINK -DBOOST_IOSTREAMS_DYN_LINK -DBOOST_THREAD_DYN_LINK)
11+
endif( )
12+
13+
# Define a single cmake project
14+
project(kenlm)
15+
16+
option(FORCE_STATIC "Build static executables" OFF)
17+
option(COMPILE_TESTS "Compile tests" OFF)
18+
option(ENABLE_PYTHON "Build Python bindings" OFF)
19+
# Eigen3 less than 3.1.0 has a race condition: http://eigen.tuxfamily.org/bz/show_bug.cgi?id=466
20+
find_package(Eigen3 3.1.0 CONFIG)
21+
include(CMakeDependentOption)
22+
cmake_dependent_option(ENABLE_INTERPOLATE "Build interpolation program (depends on Eigen3)" ON "EIGEN3_FOUND AND NOT WIN32" OFF)
23+
24+
if (FORCE_STATIC)
25+
#presumably overkill, is there a better way?
26+
#http://cmake.3232098.n2.nabble.com/Howto-compile-static-executable-td5580269.html
27+
set(Boost_USE_STATIC_LIBS ON)
28+
set_property(GLOBAL PROPERTY LINK_SEARCH_START_STATIC ON)
29+
set_property(GLOBAL PROPERTY LINK_SEARCH_END_STATIC ON)
30+
set(BUILD_SHARED_LIBRARIES OFF)
31+
if (MSVC)
32+
set(flag_vars
33+
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
34+
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO
35+
CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
36+
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO)
37+
foreach(flag_var ${flag_vars})
38+
if(${flag_var} MATCHES "/MD")
39+
string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
40+
endif(${flag_var} MATCHES "/MD")
41+
endforeach(flag_var)
42+
else (MSVC)
43+
if (NOT CMAKE_C_COMPILER_ID MATCHES ".*Clang")
44+
set(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++ -static")
45+
endif ()
46+
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
47+
endif ()
48+
#Annoyingly the exectuables say "File not found" unless these are set
49+
set(CMAKE_EXE_LINK_DYNAMIC_C_FLAGS)
50+
set(CMAKE_EXE_LINK_DYNAMIC_CXX_FLAGS)
51+
set(CMAKE_SHARED_LIBRARY_C_FLAGS)
52+
set(CMAKE_SHARED_LIBRARY_CXX_FLAGS)
53+
set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS)
54+
set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS)
55+
endif ()
56+
57+
# Compile all executables into bin/
58+
set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
59+
60+
# Compile all libraries into lib/
61+
set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
62+
63+
if (NOT CMAKE_BUILD_TYPE)
64+
set(CMAKE_BUILD_TYPE Release)
65+
endif()
66+
67+
if (COMPILE_TESTS)
68+
# Tell cmake that we want unit tests to be compiled
69+
include(CTest)
70+
enable_testing()
71+
endif()
72+
73+
# Add our CMake helper functions
74+
include(cmake/KenLMFunctions.cmake)
75+
76+
if(MSVC)
77+
set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} /w34716")
78+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /w34716")
79+
endif()
80+
81+
# And our helper modules
82+
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules)
83+
84+
# We need boost
85+
find_package(Boost 1.41.0 REQUIRED COMPONENTS
86+
program_options
87+
system
88+
thread
89+
unit_test_framework
90+
)
91+
92+
# Define where include files live
93+
include_directories(
94+
${PROJECT_SOURCE_DIR}
95+
${Boost_INCLUDE_DIRS}
96+
)
97+
98+
set(THREADS_PREFER_PTHREAD_FLAG ON)
99+
find_package(Threads REQUIRED)
100+
101+
# Process subdirectories
102+
add_subdirectory(util)
103+
add_subdirectory(lm)
104+
105+
foreach(SUBDIR IN ITEMS util util/double-conversion util/stream lm lm/builder lm/common lm/filter lm/interpolate)
106+
file(GLOB HEADERS ${CMAKE_CURRENT_LIST_DIR}/${SUBDIR}/*.h ${CMAKE_CURRENT_LIST_DIR}/${SUBDIR}/*.hh)
107+
install(FILES ${HEADERS} DESTINATION include/kenlm/${SUBDIR})
108+
endforeach(SUBDIR)
109+
110+
if(ENABLE_PYTHON)
111+
add_subdirectory(python)
112+
endif()
113+

0 commit comments

Comments
 (0)