Skip to content

Feature/cmake #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 32 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
6fee8e2
Added some CMake functionality, missing XGPU, sigproc, and fitsio dep…
cpviolator Jun 5, 2024
693803a
Add XGPU dependency, used local fork for now
cpviolator Jun 5, 2024
bb142b1
mid cmake upgrade
cpviolator Jun 5, 2024
18352cf
WAR for pthread detection
Jun 14, 2024
4082f9b
Create CMakeLists for automatic linking and building of dependencies
Jun 16, 2024
3ce3871
Move headers to include directory
Jun 16, 2024
8a50bd4
Move some code to a legacy folder, exempt from installation
Jun 16, 2024
5e0ea26
Move headers
Jun 16, 2024
7aca2bc
Remove executables
Jun 16, 2024
9fd4c10
Add CUTLASS example
Jun 16, 2024
6c35323
Remove code from src
Jun 16, 2024
75ee37b
Remove backups
Jun 16, 2024
0cad89c
Remove CPM from CMakeLists
Jun 16, 2024
03b30e9
clean up, add utils to install
Jun 16, 2024
e260867
Add more dependencies, clean CMake defaults to cuBLAS only
Jun 21, 2024
0656729
Add test directory, googletest, rename file
Jun 22, 2024
a50da91
Restructure headers, create dsalib, move executable out to tests
Jun 22, 2024
8555574
Begin work on fully arch-independent correlator and beamformer, add a…
Jun 22, 2024
b7789e2
Rename psrdada header file, split beamformer and correlator files (co…
Jun 24, 2024
f3a7c7c
Major code refactorisation
Jun 26, 2024
047722f
build tweaks
Jun 26, 2024
5a66577
Fix merge conflicts
Jun 26, 2024
95f7512
fix bug in beamformer, add util to inspect char length data in test
Jun 26, 2024
4a9b7b1
New correlator function reproduces legacy implementation data. Added …
Jun 28, 2024
6a6b6d6
Resolve merge conflict
Jun 28, 2024
bd40360
split dmem into dmem_corr and dmem_bf, add metrics structure, abstrac…
Jun 28, 2024
a7ce185
Created a Correlator class to allow for persistent memory, added miss…
Jun 29, 2024
41a8ee9
remove timer download, include header only
Jun 29, 2024
fd273c3
Implemented overlapping comms and compute for the Correlator class
Jul 5, 2024
48d6b6d
Rename files for convenient auto complete in CL
Jul 5, 2024
c422a86
Add untracked files
Jul 5, 2024
6b1600e
mid merge
Jul 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
321 changes: 321 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,321 @@
####################################################################################
# START 1. Basic setup for cmake
####################################################################################
# basic setup for cmake
cmake_minimum_required(VERSION 3.18 FATAL_ERROR)

if(POLICY CMP0074)
cmake_policy(SET CMP0074 NEW)
endif()

set(CMAKE_INCLUDE_CURRENT_DIR ON)
set(CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE ON)
set(CMAKE_COLOR_MAKEFILE ON)
set(CMAKE_CXX_STANDARD_REQUIRED True)
# Disable gnu exentions
set(CMAKE_CXX_EXTENSIONS ON)

# Define the project
project("DSA_XENGINE" VERSION 1.0.0 LANGUAGES C CXX)

# For GCC 8 and lower, set -pthread flag manually
set(CMAKE_C_FLAGS "-pthread")
set(CMAKE_CXX_FLAGS "-pthread")

# add a directory for cmake modules
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")

# DSA_XENGINE may be built to run using CUDA or CPU. Future version may be
# written for HIP or SYCL, which we call the
# Target type. By default, the target is CUDA.
#---------------------------------------------

# Set by environment variable if visible
if(DEFINED ENV{DSA_XENGINE_TARGET})
set(DEFTARGET $ENV{DSA_XENGINE_TARGET})
else()
set(DEFTARGET "CUDA")
endif()

set(VALID_TARGET_TYPES CUDA CPU) #HIP SYCL
set(DSA_XENGINE_TARGET_TYPE "${DEFTARGET}" CACHE STRING "Choose the type of target, options are: ${VALID_TARGET_TYPES}")
set_property(CACHE DSA_XENGINE_TARGET_TYPE PROPERTY STRINGS "CUDA" "CPU") # HIP SYCL

string(TOUPPER ${DSA_XENGINE_TARGET_TYPE} CHECK_TARGET_TYPE)
list(FIND VALID_TARGET_TYPES ${CHECK_TARGET_TYPE} TARGET_TYPE_VALID)

if(TARGET_TYPE_VALID LESS 0)
message(SEND_ERROR "Please specify a valid DSA_XENGINE_TARGET_TYPE type! Valid target types are:" "${VALID_TARGET_TYPES}")
endif()

# Git helpers
#------------
find_package(Git)
if(GIT_FOUND)
execute_process(
COMMAND ${GIT_EXECUTABLE} show
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
RESULT_VARIABLE IS_GIT_REPOSIITORY
OUTPUT_QUIET ERROR_QUIET)
if(${IS_GIT_REPOSIITORY} EQUAL 0)
execute_process(
COMMAND ${GIT_EXECUTABLE} describe --abbrev=0
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
OUTPUT_VARIABLE GITTAG
OUTPUT_STRIP_TRAILING_WHITESPACE)
# we use git rev-list and pipe that through wc here. Newer git versions support --count as option to rev-list but
# that might not always be available
execute_process(
COMMAND ${GIT_EXECUTABLE} rev-list ${GITTAG}..HEAD
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
COMMAND wc -l
OUTPUT_VARIABLE GITCOUNT
OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(
COMMAND ${GIT_EXECUTABLE} describe --match 1 --always --long --dirty
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
OUTPUT_VARIABLE GITVERSION
OUTPUT_STRIP_TRAILING_WHITESPACE)
endif()
endif(GIT_FOUND)


option(DSA_XENGINE_BUILD_ALL_TESTS "build tests by default" ON)
option(DSA_XENGINE_INSTALL_ALL_TESTS "install tests by default" ON)
option(DSA_XENGINE_BUILD_SHAREDLIB "build dsaXengine as a shared lib" ON)


# Use ExternalProject_Add for libtcc (borks with FetchContent)
# Use ExternalProject_Add for CUTLASS (long build time, version 2.11.0 for sm_8x arch)
include(ExternalProject)

# Use FetchContent for lightweight dependencies
include(FetchContent)

# CUDA based dependencies and options
#------------------------------------
if(DSA_XENGINE_TARGET_TYPE STREQUAL CUDA)

# CUDA specific part of CMakeLists
enable_language(CUDA)
find_package(CUDAToolkit REQUIRED)

# Get GPU architecture from environmen, or set default (sm_80)
if(DEFINED ENV{DSA_XENGINE_GPU_ARCH})
set(DSA_XENGINE_DEFAULT_GPU_ARCH $ENV{DSA_XENGINE_GPU_ARCH})
else()
set(DSA_XENGINE_DEFAULT_GPU_ARCH sm_80)
endif()

if(NOT DSA_XENGINE_GPU_ARCH)
message(STATUS "Building DSA_XENGINE for GPU ARCH " "${DSA_XENGINE_DEFAULT_GPU_ARCH}")
endif()

set(DSA_XENGINE_GPU_ARCH
${DSA_XENGINE_DEFAULT_GPU_ARCH}
CACHE STRING "set the GPU architecture (sm_60, sm_70, sm_80 sm_90)")
set_property(CACHE DSA_XENGINE_GPU_ARCH PROPERTY STRINGS sm_60 sm_70 sm_80 sm_90)
set(DSA_XENGINE_GPU_ARCH_SUFFIX
""
CACHE STRING "set the GPU architecture suffix (virtual, real). Leave empty for no suffix.")
set_property(CACHE DSA_XENGINE_GPU_ARCH_SUFFIX PROPERTY STRINGS "real" "virtual" " ")
#set(CMAKE_CUDA_ARCHITECTURES ${DSA_XENGINE_GPU_ARCH})
#mark_as_advanced(DSA_XENGINE_GPU_ARCH_SUFFIX)
#mark_as_advanced(CMAKE_CUDA_ARCHITECTURES)

# Set CUDA based methods and dependencies
#----------------------------------------

# This is the default GPU method
option(DSA_XENGINE_ENABLE_CUBLAS "Use cuBLAS for correlatorss" ON)

# All other GPU methods can be enabled at compile time and
# toggled for use at run time, if enabled.

# Get TCC dependency
option(DSA_XENGINE_ENABLE_TCC "Use TensorCoreCorrelators for correlatorss" OFF)
if(DSA_XENGINE_ENABLE_TCC)
add_compile_definitions(DSA_XENGINE_ENABLE_TCC)
option(DSA_XENGINE_DOWNLOAD_TCC "Download, build, link (and install) TCC" OFF)
if(DSA_XENGINE_DOWNLOAD_TCC)
ExternalProject_Add(TCC
GIT_REPOSITORY https://git.astron.nl/RD/tensor-core-correlator
#GIT_TAG 11d8a4a504d7073a2a33b81e1e387b12e58a420c
CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
)
else()
find_package(libtcc REQUIRED)
endif()
endif()

# Get CUTLASS dependency
option(DSA_XENGINE_ENABLE_CUTLASS "Use CUTLASS for GEMMs" OFF)
if(DSA_XENGINE_ENABLE_CUTLASS)
add_compile_definitions(DSA_XENGINE_ENABLE_CUTLASS)
option(DSA_XENGINE_DOWNLOAD_CUTLASS "Download, build (only the required kernels) link (and install) CUTLASS" OFF)
if(DSA_XENGINE_DOWNLOAD_CUTLASS)
# Custom CUTLASS build
ExternalProject_Add(NvidiaCutlass
GIT_REPOSITORY https://github.com/NVIDIA/cutlass.git
GIT_TAG 7d49e6c7e2f8896c47f586706e67e1fb215529dc
CMAKE_ARGS
"-DCUTLASS_NVCC_ARCHS_ENABLED=89"
"-DCUTLASS_LIBRARY_KERNELS=cutlass_tensorop_*gemm_planar_complex"
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
)
else()
find_package(NvidiaCutlass REQUIRED)
endif()
endif()

# Get MAGMA dependency
option(DSA_XENGINE_ENABLE_MAGMA "Use MAGMA for GEMMs" OFF)
if(DSA_XENGINE_ENABLE_MAGMA)
add_compile_definitions(DSA_XENGINE_ENABLE_MAGMA)
option(DSA_XENGINE_DOWNLOAD_MAGMA "Download, build (only the required kernels) link (and install) MAGMA" OFF)
if(DSA_XENGINE_DOWNLOAD_MAGMA)
# Custom MAGMA build
ExternalProject_Add(Magma
URL https://icl.utk.edu/projectsfiles/magma/downloads/magma-2.8.0.tar.gz
CMAKE_ARGS
"-DMAGMA_ENABLE_CUDA=ON"
"-DGPU_TARGET=sm_80"
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
)
else()
find_package(Magma REQUIRED)
endif()
endif()

# Get XGPU dependency (fix install)
option(DSA_XENGINE_ENABLE_XGPU "Use xGPU for correlatorss" OFF)
if(DSA_XENGINE_ENABLE_XGPU)
add_compile_definitions(DSA_XENGINE_ENABLE_XGPU)
option(DSA_XENGINE_DOWNLOAD_XGPU "Download and build xGPU" OFF)
if(DSA_XENGINE_DOWNLOAD_XGPU)
# Download, build and install
FetchContent_Declare(
xGPU
GIT_REPOSITORY https://github.com/cpviolator/xGPU.git
#GIT_TAG 13b7fff1eac497236eb9c38e179aed3b532a88f2
)
FetchContent_MakeAvailable(XGPU)
else()
# Find and link to local install
find_package(xGPU REQUIRED)
endif()
endif()

endif() # CUDA functionality

# Get CPU based dependencies
# Get OPENBLAS dependency
option(DSA_XENGINE_ENABLE_OPENBLAS "Use OPENBLAS for GEMMs" OFF)
if(DSA_XENGINE_ENABLE_OPENBLAS)
add_compile_definitions(DSA_XENGINE_ENABLE_OPENBLAS)
option(DSA_XENGINE_DOWNLOAD_OPENBLAS "Download, build, link, and install OPENBLAS" OFF)
if(DSA_XENGINE_DOWNLOAD_OPENBLAS)
# Custom OPENBLAS build
ExternalProject_Add(Openblas
GIT_REPOSITORY https://github.com/OpenMathLib/OpenBLAS.git
GIT_TAG ce3f668
CMAKE_ARGS
#"-DOPENBLAS_ENABLE_CUDA=ON"
#"-DGPU_TARGET=sm_80"
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
)
else()
find_package(Openblas REQUIRED)
endif()
endif()

# Get psrdada dependency
option(DSA_XENGINE_ENABLE_PSRDADA "Use PSRDada for IO" ON)
option(DSA_XENGINE_DOWNLOAD_PSRDADA "Download and build PSRDada" ON)
if(DSA_XENGINE_DOWNLOAD_PSRDADA)
# Download, build and install
FetchContent_Declare(
PSRDada
GIT_REPOSITORY git://git.code.sf.net/p/psrdada/code
)
FetchContent_MakeAvailable(PSRDada)
else()
# Find and link to local install
find_package(PSRDada REQUIRED)
endif()

# Get HDF5 dependency
option(DSA_XENGINE_ENABLE_HDF5 "Use HDF5 for data IO" OFF)
if(DSA_XENGINE_ENABLE_HDF5)
option(DSA_XENGINE_DOWNLOAD_HDF5 "Download and build HDf5" OFF)
if(DSA_XENGINE_DOWNLOAD_HDF5)
# Download, build and install
FetchContent_Declare(
HDF5
GIT_REPOSITORY https://github.com/HDFGroup/hdf5.git
GIT_TAG 5794814
)
FetchContent_MakeAvailable(HDF5)
else()
# Find and link to local install
find_package(HDF5 REQUIRED)
endif()
endif()

# Get CLI11 dependency
# FIX ME: get static .hpp version and ship with package
option(DSA_XENGINE_ENABLE_CLI11 "Enable CLI11 (required)" ON)
if(DSA_XENGINE_ENABLE_CLI11)
option(DSA_XENGINE_DOWNLOAD_CLI11 "Download and build CLI11" ON)
if(DSA_XENGINE_DOWNLOAD_CLI11)
# Download, build and install
FetchContent_Declare(
CLI11
GIT_REPOSITORY https://github.com/CLIUtils/CLI11.git
GIT_TAG main
)
FetchContent_MakeAvailable(CLI11)
else()
# Find and link to local install
find_package(CLI11 REQUIRED)
endif()
endif()


# Get ZFP dependency
option(DSA_XENGINE_ENABLE_ZFP "Enable ZFP" OFF)
if(DSA_XENGINE_ENABLE_ZFP)
option(DSA_XENGINE_DOWNLOAD_ZFP "Download and build ZFP" OFF)
if(DSA_XENGINE_DOWNLOAD_ZFP)
# Download, build and install
FetchContent_Declare(
ZFP
GIT_REPOSITORY https://github.com/LLNL/zfp.git
GIT_TAG f40868a
)
FetchContent_MakeAvailable(ZFP)
else()
# Find and link to local install
find_package(ZFP REQUIRED)
endif()
endif()

# Add src, include, tests, and legacy
add_subdirectory(src)
add_subdirectory(include)
add_subdirectory(tests)
option(DSA_XENGINE_BUILD_LEGACY "Build legacy code (will not install if built)" OFF)
if(DSA_XENGINE_BUILD_LEGACY)
add_subdirectory(legacy)
endif()

# Install project cmake targets
include(CMakePackageConfigHelpers)
write_basic_package_version_file(
${PROJECT_NAME}-config-version.cmake
VERSION ${DSA_XENGINE_VERSION}
COMPATIBILITY AnyNewerVersion
)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
)
13 changes: 5 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# dsa110-xengine

<p align="center">
<a href="https://github.com/dsa110/dsa110-xengine/commits/master"><img src="https://img.shields.io/github/last-commit/dsa110/dsa110-xengine" alt="GitHub last commit"></a>
<a href="https://github.com/dsa110/dsa110-xengine/commits/master"><img src="https://img.shields.io/github/commit-activity/y/dsa110/dsa110-xengine" alt="GitHub commit activity the past week"></a>
</p>

This repo contains code used for the DSA X-engine. The requirements are to:
- capture SNAP F-engine packets on an ethernet interface, and place them in a psrdada buffer
Expand Down Expand Up @@ -67,11 +71,4 @@ Finally, `dsaX_dbnic` and `dsaX_nicdb` implement the corner turn to feed `mbheim

### scripts and utils

The "scripts" dir contains some useful scripts to test various aspects of the system (corr, bf, cornerturn). The "utils" dir includes functionality to generate fake data and beamforming weights.







The "scripts" dir contains some useful scripts to test various aspects of the system (corr, bf, cornerturn). The "utils" dir includes functionality to generate fake data and beamforming weights.
21 changes: 21 additions & 0 deletions include/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
enable_language(CUDA)

# install step for header files
#------------------------------
set(DSA_XENGINE_HEADERS
# cmake-format: sortable
dsaX.h
dsaX_def.h
dsaX_malloc.h
dsaX_ptr.h
fast_time_domain.h
cuda_interface.h
cuda_handles.h
cuda_headers.h
dsaX_capture.h
dsaX_capture_manythread.h
dsaX_capture_pcap.h
cutlass_interface.h
)
install(FILES ${DSA_XENGINE_HEADERS} DESTINATION include)
#------------------------------
5 changes: 5 additions & 0 deletions include/blas_interface.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#pragma once

#include "interface.h"

void dsaXHgemmStridedBatched(void *real_a, void *imag_a, void *real_b, void *imag_b, void *real_c, void *imag_c, dsaXBLASParam param, int stream = 0);
4 changes: 4 additions & 0 deletions include/cublas_interface.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#pragma once
#include "dsaX.h"

void dsaXHgemmStridedBatchedCuda(void *real_a, void *imag_a, void *real_b, void *imag_b, void *real_c, void *imag_c, dsaXBLASParam param, int stream);
20 changes: 20 additions & 0 deletions include/cuda_handles.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#pragma once

#include <vector>

#include "utils.h"

#ifdef DSA_XENGINE_TARGET_CUDA
#include "cuda_headers.h"

static std::vector<cudaStream_t> streams;
static cublasHandle_t cublasH = NULL;

static bool cublas_init = false;
static bool stream_init = false;

cudaStream_t get_stream(unsigned int i);
#endif

void init_streams(unsigned int n_streams);
void destroy_streams();
Loading