diff --git a/.circleci/README.md b/.circleci/README.md new file mode 100644 index 000000000000..8de411c5a915 --- /dev/null +++ b/.circleci/README.md @@ -0,0 +1,89 @@ +CircleCi integration is controlled by the `./circleci/config.yml` file. Our +config currently contains two workflows. One is triggered on every pull request update. +The other workflow runs nightly to verify our compatibility with prestodb internal protocol. + +The PR workflow is named `dist-compile` and has 4 jobs, 2 to build and run unit tests on linux and macos +and 2 to check code formatting and license headers: +* linux-build +* macos-build +* format-check +* header-check + +## Running locally + +The linux container based jobs can be run locally using the `circleci` cli: + +``` + circleci local execute --job JOB_NAME +``` + +For example to run unit tests use: + +``` + circleci local execute --job linux-build +``` + +A Nightly build with prestodb/master sync checks that the presto_protocol library +remains in sync with Presto Java. + +Run the nightly sync job locally: +``` + circleci local execute --job presto-sync +``` + +## Install CircleCi cli +``` + curl -fLSs https://circle.ci/cli | bash +``` + +To use containers Docker must be installed. Here are instructions to [Install +Docker on macos](https://docs.docker.com/docker-for-mac/install/). Docker deamon +must be running before issuing the `circleci` commands. + +### Macos testing + +Macos testing is done by using the CircleCi macos executor and installing +dependencies each time the job is run. This executor cannot be run locally. +The script `scripts/setup-macos.sh` contains commands that are run as part of +this job to install these dependencies. + +### Linux testing + +Linux testing uses a Docker container. The container build depends on the Velox CircleCi container. Check +f4d/.circleci/config.yml to see that the base container in circleci-container.dockfile is using the latest. +The container build uses Docker and should be run on your macos or linux laptop with Docker installed and +running. + +#### Build the base container: + +* In an up-to-date clone of f4d (maybe you have one?) + +``` +git clone git@github.com:facebookexternal/f4d.git +cd f4d +make base-container +``` +* Wait - This step takes rather a long time. It is building clang-format v8 to be compatible with fbcode +* When the base container is finished the new container name will be printed on the console. +* Push the container to DockerHub +``` +docker push prestocpp/base-container:$USER-YYYYMMDD +``` +* After the push, update `scripts/velox-container.dockfile` with the newly build base container name + +#### Build the dependencies container + +* If you have a new base-container update scripts/velox-container.dockfile to refer to it +* Build the velox container +``` +make velox-container.dockfile +``` +* Wait - This takes a few minutes, but not nearly as long as the base container. +* When the velox container is finished the new container name will be printed on the console. +* Push the container to DockerHub +``` +docker push prestocpp/velox-container:$USER-YYYYMMDD +``` +* Update `.circleci/config.yml` with the newly built circleci container name. + There are two places in the config.yml file that refer to the container, update + both. diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000000..5b762ea2a440 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,130 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +version: 2.1 +workflows: + version: 2 + dist-compile: + jobs: + - linux-build + - macos-build + - format-check + - header-check + +executors: + build: + docker: + - image : prestocpp/velox-circleci:mikesh-20210610 + resource_class: xlarge + environment: + CC: /opt/rh/gcc-toolset-9/root/bin/gcc + CXX: /opt/rh/gcc-toolset-9/root/bin/g++ + check: + docker: + - image : prestocpp/velox-check:mikesh-20210609 + +jobs: + macos-build: + macos: + xcode: "11.7.0" + steps: + - checkout + - restore_cache: + name: "Restore Dependency Cache" + key: velox-circleci-macos-deps-{{ checksum ".circleci/config.yml" }}-{{ checksum "scripts/setup-macos.sh" }} + - run: + name: "Install dependencies" + command: | + set -xu + if [[ ! -e ~/deps ]]; then + mkdir ~/deps ~/deps-src + curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C ~/deps + PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-macos.sh + rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%. + fi + - save_cache: + name: "Save Dependency Cache" + key: velox-circleci-macos-deps-{{ checksum ".circleci/config.yml" }}-{{ checksum "scripts/setup-macos.sh" }} + paths: + - ~/deps + - run: + name: "Calculate merge-base for CCache" + command: git merge-base origin/master HEAD > merge-base + - restore_cache: + name: "Restore CCache cache" + keys: + - velox-ccache-{{ arch }}-{{ checksum "merge-base" }} + - run: + name: "Build on MacOS" + command: | + export PATH=~/deps/bin:${PATH} + mkdir -p .ccache + export CCACHE_DIR=$(pwd)/.ccache + ccache -sz -M 5Gi + cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=~/deps -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + ninja -C _build/debug + ccache -s + no_output_timeout: 1h + - save_cache: + name: "Save CCache cache" + key: velox-ccache-{{ arch }}-{{ checksum "merge-base" }} + paths: + - .ccache/ + + linux-build: + executor: build + steps: + - checkout + - run: + name: "Calculate merge-base for CCache" + command: git merge-base origin/master HEAD > merge-base + - restore_cache: + name: "Restore CCache cache" + keys: + - velox-ccache-{{ arch }}-{{ checksum "merge-base" }} + - run: + name: Build + command: | + mkdir -p .ccache + export CCACHE_DIR=$(realpath .ccache) + ccache -sz -M 5Gi + source /opt/rh/gcc-toolset-9/enable + make debug NUM_THREADS=8 MAX_HIGH_MEM_JOBS=3 MAX_LINK_JOBS=4 + ccache -s + no_output_timeout: 1h + - store_artifacts: + path: '_build/debug/.ninja_log' + - run: + name: Run Unit Tests + command: | + make unittest NUM_THREADS=8 + no_output_timeout: 1h + - save_cache: + name: "Save CCache cache" + key: velox-ccache-{{ arch }}-{{ checksum "merge-base" }} + paths: + - .ccache/ + + format-check: + executor: check + steps: + - checkout + - run: + name: Check formatting + command: 'make format-check' + + header-check: + executor: check + steps: + - checkout + - run: + name: Check license headers + command: 'make header-check' diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000000..eab4576fe09a --- /dev/null +++ b/.clang-format @@ -0,0 +1,87 @@ +--- +AccessModifierOffset: -1 +AlignAfterOpenBracket: AlwaysBreak +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlinesLeft: true +AlignOperands: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: false +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ForEachMacros: [ FOR_EACH, FOR_EACH_R, FOR_EACH_RANGE, ] +IncludeCategories: + - Regex: '^<.*\.h(pp)?>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IndentCaseLabels: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 8 +UseTab: Never +... diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..fc325ab88271 --- /dev/null +++ b/.gitignore @@ -0,0 +1,311 @@ +#==============================================================================# +# This file specifies intentionally untracked files that git should ignore. +#==============================================================================# + +#==============================================================================# +# File extensions to be ignored anywhere in the tree. +#==============================================================================# +# Temp files created by most text editors. +*~ +# Merge files created by git. +*.orig +# Java bytecode +*.class +# Byte compiled python modules. +*.pyc +# vim swap files +.*.sw? +.sw? +#OS X specific files. +.DS_store +# Core files +#core + +#==============================================================================# +# Explicit files to ignore (only matches one). +#==============================================================================# +# Various tag programs +/tags +/TAGS +/GPATH +/GRTAGS +/GSYMS +/GTAGS +.gitusers +autom4te.cache +cscope.files +cscope.out +autoconf/aclocal.m4 +autoconf/autom4te.cache +/compile_commands.json + +#==============================================================================# +# Directories to ignore (do not add trailing '/'s, they skip symlinks). +#==============================================================================# +# External projects that are tracked independently. +projects/* +!projects/*.* +!projects/Makefile + + +#==============================================================================# +# Autotools artifacts +#==============================================================================# +config/ +configure +config-h.in +autom4te.cache +*Makefile.in +third_party/*/Makefile +libtool +aclocal.m4 +config.log +config.status +stamp-h1 +config.h +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 + +#==============================================================================# +# Build artifacts +#==============================================================================# +#m4/ +build/ +_build/ +#*.m4 +*.o +*.lo +*.la +*~ +*.pdf +*.swp +a.out + +#==============================================================================# +# Kate Swap Files +#==============================================================================# +*.kate-swp +.#kate-* + +#==============================================================================# +# Backup artifacts +#==============================================================================# +~* +*~ +tmp/ + +#==============================================================================# +# KDevelop files +#==============================================================================# +.kdev4 +*.kdev4 +.dirstamp +.deps +.libs + +#==============================================================================# +# Eclipse files +#==============================================================================# +.wtpmodules +.classpath +.project +.cproject +.pydevproject +.settings +.autotools +.csettings + +/Debug/ +/misc/ + +#==============================================================================# +# Intellij files +#==============================================================================# +.idea +*.iml + +#==============================================================================# +# Code Coverage files +#==============================================================================# +*.gcno +*.gcda + +#==============================================================================# +# Scripts +#==============================================================================# +*.jar +scripts/PelotonTest/out +scripts/PelotonTest/lib + +#==============================================================================# +# Protobuf +#==============================================================================# +*.pb-c.c +*.pb-c.h +*.pb.cc +*.pb.h +*.pb.go + +#==============================================================================# +# Third party +#==============================================================================# +third_party/nanomsg/ +third_party/nvml/ +third_party/logcabin/ + +#==============================================================================# +# Eclipse +#==============================================================================# + +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# Eclipse Core +.project + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# JDT-specific (Eclipse Java Development Tools) +.classpath + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ +io_file + +## General + +# Compiled Object files +*.slo +*.lo +*.o +*.cuo + +# Compiled Dynamic libraries +*.so +*.dylib + +# Compiled Static libraries +*.lai +*.la +*.a + +# Compiled protocol buffers +*.pb.h +*.pb.cc +*_pb2.py + +# Compiled python +*.pyc + +# Compiled MATLAB +*.mex* + +# IPython notebook checkpoints +.ipynb_checkpoints + +# Editor temporaries +*.swp +*~ + +# Sublime Text settings +*.sublime-workspace +*.sublime-project + +# Eclipse Project settings +*.*project +.settings +.csettings + +# Visual Studio +.vs +settings.json +.vscode + +# QtCreator files +*.user + +# PyCharm files +.idea + +# OSX dir files +.DS_Store + +# User's build configuration +Makefile.config + +# build, distribute, and bins (+ python proto bindings) +build +.build_debug/* +.build_release/* +distribute/* +*.testbin +*.bin +cmake_build +.cmake_build +cmake-build-debug +cmake-build-release + +# tests +test/test.sql + +# SQLite logic tests +test/evidence/ +third_party/sqllogictest + +#imdb dataset +third_party/imdb/data + +# Format timer +.last_format +# Benchmarks +.last_benchmarked_commit +benchmark_results/ +duckdb_unittest_tempdir/ +grammar.y.tmp +src/amalgamation/ + +#eclipse +.project +.cproject +.settings +~ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..d2bb6077817d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Facebook projects that use `fbcode_builder` for continuous integration +# share this Travis configuration to run builds via Docker. + +# Docker disables IPv6 in containers by default. Enable it for unit tests that need [::1]. +before_script: + - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; + then + sudo build/fbcode_builder/docker_enable_ipv6.sh; + fi + +env: + global: + - travis_cache_dir=$HOME/travis_ccache + # Travis times out after 50 minutes. Very generously leave 10 minutes + # for setup (e.g. cache download, compression, and upload), so we never + # fail to cache the progress we made. + - docker_build_timeout=40m + +cache: + # Our build caches can be 200-300MB, so increase the timeout to 7 minutes + # to make sure we never fail to cache the progress we made. + timeout: 420 + directories: + - $HOME/travis_ccache # see docker_build_with_ccache.sh + +# Ugh, `services:` must be in the matrix, or we get `docker: command not found` +# https://github.com/travis-ci/travis-ci/issues/5142 +matrix: + include: + - env: ['os_image=ubuntu:18.04', gcc_version=7] + services: [docker] + +addons: + apt: + packages: python2.7 + +script: + # We don't want to write the script inline because of Travis kludginess -- + # it looks like it escapes " and \ in scripts when using `matrix:`. + - ./build/fbcode_builder/travis_docker_build.sh diff --git a/CMake/FindICU.cmake b/CMake/FindICU.cmake new file mode 100644 index 000000000000..479e803bc820 --- /dev/null +++ b/CMake/FindICU.cmake @@ -0,0 +1,314 @@ +# https://raw.githubusercontent.com/KaOSx/user-kcm/master/FindICU.cmake +# This module can find the International Components for Unicode (ICU) Library +# +# Requirements: +# - CMake >= 2.8.3 (for new version of find_package_handle_standard_args) +# +# The following variables will be defined for your use: +# - ICU_FOUND : were all of your specified components found (include dependencies)? +# - ICU_INCLUDE_DIRS : ICU include directory +# - ICU_LIBRARIES : ICU libraries +# - ICU_VERSION : complete version of ICU (x.y.z) +# - ICU_MAJOR_VERSION : major version of ICU +# - ICU_MINOR_VERSION : minor version of ICU +# - ICU_PATCH_VERSION : patch version of ICU +# - ICU__FOUND : were found? (FALSE for non specified component if it is not a dependency) +# +# For windows or non standard installation, define ICU_ROOT variable to point to the root installation of ICU. Two ways: +# - run cmake with -DICU_ROOT= +# - define an environment variable with the same name before running cmake +# With cmake-gui, before pressing "Configure": +# 1) Press "Add Entry" button +# 2) Add a new entry defined as: +# - Name: ICU_ROOT +# - Type: choose PATH in the selection list +# - Press "..." button and select the root installation of ICU +# +# Example Usage: +# +# 1. Copy this file in the root of your project source directory +# 2. Then, tell CMake to search this non-standard module in your project directory by adding to your CMakeLists.txt: +# set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}) +# 3. Finally call find_package() once, here are some examples to pick from +# +# Require ICU 4.4 or later +# find_package(ICU 4.4 REQUIRED) +# +# if(ICU_FOUND) +# include_directories(${ICU_INCLUDE_DIRS}) +# add_executable(myapp myapp.c) +# target_link_libraries(myapp ${ICU_LIBRARIES}) +# endif(ICU_FOUND) + +#============================================================================= +# Copyright (c) 2011-2013, julp +# +# Distributed under the OSI-approved BSD License +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#============================================================================= + +find_package(PkgConfig QUIET) + +########## Private ########## +if(NOT DEFINED ICU_PUBLIC_VAR_NS) + set(ICU_PUBLIC_VAR_NS "ICU") # Prefix for all ICU relative public variables +endif(NOT DEFINED ICU_PUBLIC_VAR_NS) +if(NOT DEFINED ICU_PRIVATE_VAR_NS) + set(ICU_PRIVATE_VAR_NS "_${ICU_PUBLIC_VAR_NS}") # Prefix for all ICU relative internal variables +endif(NOT DEFINED ICU_PRIVATE_VAR_NS) +if(NOT DEFINED PC_ICU_PRIVATE_VAR_NS) + set(PC_ICU_PRIVATE_VAR_NS "_PC${ICU_PRIVATE_VAR_NS}") # Prefix for all pkg-config relative internal variables +endif(NOT DEFINED PC_ICU_PRIVATE_VAR_NS) + +function(icudebug _VARNAME) + if(${ICU_PUBLIC_VAR_NS}_DEBUG) + if(DEFINED ${ICU_PUBLIC_VAR_NS}_${_VARNAME}) + message("${ICU_PUBLIC_VAR_NS}_${_VARNAME} = ${${ICU_PUBLIC_VAR_NS}_${_VARNAME}}") + else(DEFINED ${ICU_PUBLIC_VAR_NS}_${_VARNAME}) + message("${ICU_PUBLIC_VAR_NS}_${_VARNAME} = ") + endif(DEFINED ${ICU_PUBLIC_VAR_NS}_${_VARNAME}) + endif(${ICU_PUBLIC_VAR_NS}_DEBUG) +endfunction(icudebug) + +set(${ICU_PRIVATE_VAR_NS}_ROOT "") +if(DEFINED ENV{ICU_ROOT}) + set(${ICU_PRIVATE_VAR_NS}_ROOT "$ENV{ICU_ROOT}") +endif(DEFINED ENV{ICU_ROOT}) +if (DEFINED ICU_ROOT) + set(${ICU_PRIVATE_VAR_NS}_ROOT "${ICU_ROOT}") +endif(DEFINED ICU_ROOT) + +set(${ICU_PRIVATE_VAR_NS}_BIN_SUFFIXES ) +set(${ICU_PRIVATE_VAR_NS}_LIB_SUFFIXES ) +if(CMAKE_SIZEOF_VOID_P EQUAL 8) + list(APPEND ${ICU_PRIVATE_VAR_NS}_BIN_SUFFIXES "bin64") + list(APPEND ${ICU_PRIVATE_VAR_NS}_LIB_SUFFIXES "lib64") +endif(CMAKE_SIZEOF_VOID_P EQUAL 8) +list(APPEND ${ICU_PRIVATE_VAR_NS}_BIN_SUFFIXES "bin") +list(APPEND ${ICU_PRIVATE_VAR_NS}_LIB_SUFFIXES "lib") + +set(${ICU_PRIVATE_VAR_NS}_COMPONENTS ) +# ... +macro(icu_declare_component _NAME) + list(APPEND ${ICU_PRIVATE_VAR_NS}_COMPONENTS ${_NAME}) + set("${ICU_PRIVATE_VAR_NS}_COMPONENTS_${_NAME}" ${ARGN}) +endmacro(icu_declare_component) + +icu_declare_component(data icudata) +icu_declare_component(uc icuuc) # Common and Data libraries +icu_declare_component(i18n icui18n icuin) # Internationalization library +icu_declare_component(io icuio ustdio) # Stream and I/O Library +icu_declare_component(le icule) # Layout library +icu_declare_component(lx iculx) # Paragraph Layout library + +########## Public ########## +set(${ICU_PUBLIC_VAR_NS}_FOUND TRUE) +set(${ICU_PUBLIC_VAR_NS}_LIBRARIES ) +set(${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS ) +set(${ICU_PUBLIC_VAR_NS}_C_FLAGS "") +set(${ICU_PUBLIC_VAR_NS}_CXX_FLAGS "") +set(${ICU_PUBLIC_VAR_NS}_CPP_FLAGS "") +set(${ICU_PUBLIC_VAR_NS}_C_SHARED_FLAGS "") +set(${ICU_PUBLIC_VAR_NS}_CXX_SHARED_FLAGS "") +set(${ICU_PUBLIC_VAR_NS}_CPP_SHARED_FLAGS "") +foreach(${ICU_PRIVATE_VAR_NS}_COMPONENT ${${ICU_PRIVATE_VAR_NS}_COMPONENTS}) + string(TOUPPER "${${ICU_PRIVATE_VAR_NS}_COMPONENT}" ${ICU_PRIVATE_VAR_NS}_UPPER_COMPONENT) + set("${ICU_PUBLIC_VAR_NS}_${${ICU_PRIVATE_VAR_NS}_UPPER_COMPONENT}_FOUND" FALSE) # may be done in the icu_declare_component macro +endforeach(${ICU_PRIVATE_VAR_NS}_COMPONENT) + +# Check components +if(NOT ${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS) # uc required at least + set(${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS uc) +else(NOT ${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS) + list(APPEND ${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS uc) + list(REMOVE_DUPLICATES ${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS) + foreach(${ICU_PRIVATE_VAR_NS}_COMPONENT ${${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS}) + if(NOT DEFINED ${ICU_PRIVATE_VAR_NS}_COMPONENTS_${${ICU_PRIVATE_VAR_NS}_COMPONENT}) + message(FATAL_ERROR "Unknown ICU component: ${${ICU_PRIVATE_VAR_NS}_COMPONENT}") + endif(NOT DEFINED ${ICU_PRIVATE_VAR_NS}_COMPONENTS_${${ICU_PRIVATE_VAR_NS}_COMPONENT}) + endforeach(${ICU_PRIVATE_VAR_NS}_COMPONENT) +endif(NOT ${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS) + +# Includes +find_path( + ${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS + NAMES unicode/utypes.h utypes.h + HINTS ${${ICU_PRIVATE_VAR_NS}_ROOT} + PATH_SUFFIXES "include" + DOC "Include directories for ICU" +) + +if(${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS) + ########## ########## + if(EXISTS "${${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS}/unicode/uvernum.h") # ICU >= 4 + file(READ "${${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS}/unicode/uvernum.h" ${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS) + elseif(EXISTS "${${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS}/unicode/uversion.h") # ICU [2;4[ + file(READ "${${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS}/unicode/uversion.h" ${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS) + elseif(EXISTS "${${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS}/unicode/utypes.h") # ICU [1.4;2[ + file(READ "${${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS}/unicode/utypes.h" ${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS) + elseif(EXISTS "${${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS}/utypes.h") # ICU 1.3 + file(READ "${${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS}/utypes.h" ${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS) + else() + message(FATAL_ERROR "ICU version header not found") + endif() + + if(${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS MATCHES ".*# *define *ICU_VERSION *\"([0-9]+)\".*") # ICU 1.3 + # [1.3;1.4[ as #define ICU_VERSION "3" (no patch version, ie all 1.3.X versions will be detected as 1.3.0) + set(${ICU_PUBLIC_VAR_NS}_MAJOR_VERSION "1") + set(${ICU_PUBLIC_VAR_NS}_MINOR_VERSION "${CMAKE_MATCH_1}") + set(${ICU_PUBLIC_VAR_NS}_PATCH_VERSION "0") + elseif(${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS MATCHES ".*# *define *U_ICU_VERSION_MAJOR_NUM *([0-9]+).*") + # + # Since version 4.9.1, ICU release version numbering was totaly changed, see: + # - http://site.icu-project.org/download/49 + # - http://userguide.icu-project.org/design#TOC-Version-Numbers-in-ICU + # + set(${ICU_PUBLIC_VAR_NS}_MAJOR_VERSION "${CMAKE_MATCH_1}") + string(REGEX REPLACE ".*# *define *U_ICU_VERSION_MINOR_NUM *([0-9]+).*" "\\1" ${ICU_PUBLIC_VAR_NS}_MINOR_VERSION "${${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS}") + string(REGEX REPLACE ".*# *define *U_ICU_VERSION_PATCHLEVEL_NUM *([0-9]+).*" "\\1" ${ICU_PUBLIC_VAR_NS}_PATCH_VERSION "${${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS}") + elseif(${ICU_PRIVATE_VAR_NS}_VERSION_HEADER_CONTENTS MATCHES ".*# *define *U_ICU_VERSION *\"(([0-9]+)(\\.[0-9]+)*)\".*") # ICU [1.4;1.8[ + # [1.4;1.8[ as #define U_ICU_VERSION "1.4.1.2" but it seems that some 1.4.1(?:\.\d)? have releasing error and appears as 1.4.0 + set(${ICU_PRIVATE_VAR_NS}_FULL_VERSION "${CMAKE_MATCH_1}") # copy CMAKE_MATCH_1, no longer valid on the following if + if(${ICU_PRIVATE_VAR_NS}_FULL_VERSION MATCHES "^([0-9]+)\\.([0-9]+)$") + set(${ICU_PUBLIC_VAR_NS}_MAJOR_VERSION "${CMAKE_MATCH_1}") + set(${ICU_PUBLIC_VAR_NS}_MINOR_VERSION "${CMAKE_MATCH_2}") + set(${ICU_PUBLIC_VAR_NS}_PATCH_VERSION "0") + elseif(${ICU_PRIVATE_VAR_NS}_FULL_VERSION MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)") + set(${ICU_PUBLIC_VAR_NS}_MAJOR_VERSION "${CMAKE_MATCH_1}") + set(${ICU_PUBLIC_VAR_NS}_MINOR_VERSION "${CMAKE_MATCH_2}") + set(${ICU_PUBLIC_VAR_NS}_PATCH_VERSION "${CMAKE_MATCH_3}") + endif() + else() + message(FATAL_ERROR "failed to detect ICU version") + endif() + set(${ICU_PUBLIC_VAR_NS}_VERSION "${${ICU_PUBLIC_VAR_NS}_MAJOR_VERSION}.${${ICU_PUBLIC_VAR_NS}_MINOR_VERSION}.${${ICU_PUBLIC_VAR_NS}_PATCH_VERSION}") + ########## ########## + + # Check dependencies (implies pkg-config) + if(PKG_CONFIG_FOUND) + set(${ICU_PRIVATE_VAR_NS}_COMPONENTS_DUP ${${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS}) + foreach(${ICU_PRIVATE_VAR_NS}_COMPONENT ${${ICU_PRIVATE_VAR_NS}_COMPONENTS_DUP}) + pkg_check_modules(PC_ICU_PRIVATE_VAR_NS "icu-${${ICU_PRIVATE_VAR_NS}_COMPONENT}" QUIET) + + if(${PC_ICU_PRIVATE_VAR_NS}_FOUND) + foreach(${PC_ICU_PRIVATE_VAR_NS}_LIBRARY ${PC_ICU_LIBRARIES}) + string(REGEX REPLACE "^icu" "" ${PC_ICU_PRIVATE_VAR_NS}_STRIPPED_LIBRARY ${${PC_ICU_PRIVATE_VAR_NS}_LIBRARY}) + list(APPEND ${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS ${${PC_ICU_PRIVATE_VAR_NS}_STRIPPED_LIBRARY}) + endforeach(${PC_ICU_PRIVATE_VAR_NS}_LIBRARY) + endif(${PC_ICU_PRIVATE_VAR_NS}_FOUND) + endforeach(${ICU_PRIVATE_VAR_NS}_COMPONENT) + list(REMOVE_DUPLICATES ${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS) + endif(PKG_CONFIG_FOUND) + + # Check libraries + foreach(${ICU_PRIVATE_VAR_NS}_COMPONENT ${${ICU_PUBLIC_VAR_NS}_FIND_COMPONENTS}) + set(${ICU_PRIVATE_VAR_NS}_POSSIBLE_RELEASE_NAMES ) + set(${ICU_PRIVATE_VAR_NS}_POSSIBLE_DEBUG_NAMES ) + foreach(${ICU_PRIVATE_VAR_NS}_BASE_NAME ${${ICU_PRIVATE_VAR_NS}_COMPONENTS_${${ICU_PRIVATE_VAR_NS}_COMPONENT}}) + list(APPEND ${ICU_PRIVATE_VAR_NS}_POSSIBLE_RELEASE_NAMES "${${ICU_PRIVATE_VAR_NS}_BASE_NAME}") + list(APPEND ${ICU_PRIVATE_VAR_NS}_POSSIBLE_DEBUG_NAMES "${${ICU_PRIVATE_VAR_NS}_BASE_NAME}d") + list(APPEND ${ICU_PRIVATE_VAR_NS}_POSSIBLE_RELEASE_NAMES "${${ICU_PRIVATE_VAR_NS}_BASE_NAME}${ICU_MAJOR_VERSION}${ICU_MINOR_VERSION}") + list(APPEND ${ICU_PRIVATE_VAR_NS}_POSSIBLE_DEBUG_NAMES "${${ICU_PRIVATE_VAR_NS}_BASE_NAME}${ICU_MAJOR_VERSION}${ICU_MINOR_VERSION}d") + endforeach(${ICU_PRIVATE_VAR_NS}_BASE_NAME) + + find_library( + ${ICU_PRIVATE_VAR_NS}_LIB_RELEASE_${${ICU_PRIVATE_VAR_NS}_COMPONENT} + NAMES ${${ICU_PRIVATE_VAR_NS}_POSSIBLE_RELEASE_NAMES} + HINTS ${${ICU_PRIVATE_VAR_NS}_ROOT} + PATH_SUFFIXES ${_ICU_LIB_SUFFIXES} + DOC "Release libraries for ICU" + ) + find_library( + ${ICU_PRIVATE_VAR_NS}_LIB_DEBUG_${${ICU_PRIVATE_VAR_NS}_COMPONENT} + NAMES ${${ICU_PRIVATE_VAR_NS}_POSSIBLE_DEBUG_NAMES} + HINTS ${${ICU_PRIVATE_VAR_NS}_ROOT} + PATH_SUFFIXES ${_ICU_LIB_SUFFIXES} + DOC "Debug libraries for ICU" + ) + + string(TOUPPER "${${ICU_PRIVATE_VAR_NS}_COMPONENT}" ${ICU_PRIVATE_VAR_NS}_UPPER_COMPONENT) + if(NOT ${ICU_PRIVATE_VAR_NS}_LIB_RELEASE_${${ICU_PRIVATE_VAR_NS}_COMPONENT} AND NOT ${ICU_PRIVATE_VAR_NS}_LIB_DEBUG_${${ICU_PRIVATE_VAR_NS}_COMPONENT}) # both not found + set("${ICU_PUBLIC_VAR_NS}_${${ICU_PRIVATE_VAR_NS}_UPPER_COMPONENT}_FOUND" FALSE) + set("${ICU_PUBLIC_VAR_NS}_FOUND" FALSE) + else(NOT ${ICU_PRIVATE_VAR_NS}_LIB_RELEASE_${${ICU_PRIVATE_VAR_NS}_COMPONENT} AND NOT ${ICU_PRIVATE_VAR_NS}_LIB_DEBUG_${${ICU_PRIVATE_VAR_NS}_COMPONENT}) # one or both found + set("${ICU_PUBLIC_VAR_NS}_${${ICU_PRIVATE_VAR_NS}_UPPER_COMPONENT}_FOUND" TRUE) + if(NOT ${ICU_PRIVATE_VAR_NS}_LIB_RELEASE_${${ICU_PRIVATE_VAR_NS}_COMPONENT}) # release not found => we are in debug + set(${ICU_PRIVATE_VAR_NS}_LIB_${${ICU_PRIVATE_VAR_NS}_COMPONENT} "${${ICU_PRIVATE_VAR_NS}_LIB_DEBUG_${${ICU_PRIVATE_VAR_NS}_COMPONENT}}") + elseif(NOT ${ICU_PRIVATE_VAR_NS}_LIB_DEBUG_${${ICU_PRIVATE_VAR_NS}_COMPONENT}) # debug not found => we are in release + set(${ICU_PRIVATE_VAR_NS}_LIB_${${ICU_PRIVATE_VAR_NS}_COMPONENT} "${${ICU_PRIVATE_VAR_NS}_LIB_RELEASE_${${ICU_PRIVATE_VAR_NS}_COMPONENT}}") + else() # both found + set( + ${ICU_PRIVATE_VAR_NS}_LIB_${${ICU_PRIVATE_VAR_NS}_COMPONENT} + optimized ${${ICU_PRIVATE_VAR_NS}_LIB_RELEASE_${${ICU_PRIVATE_VAR_NS}_COMPONENT}} + debug ${${ICU_PRIVATE_VAR_NS}_LIB_DEBUG_${${ICU_PRIVATE_VAR_NS}_COMPONENT}} + ) + endif() + list(APPEND ${ICU_PUBLIC_VAR_NS}_LIBRARIES ${${ICU_PRIVATE_VAR_NS}_LIB_${${ICU_PRIVATE_VAR_NS}_COMPONENT}}) + endif(NOT ${ICU_PRIVATE_VAR_NS}_LIB_RELEASE_${${ICU_PRIVATE_VAR_NS}_COMPONENT} AND NOT ${ICU_PRIVATE_VAR_NS}_LIB_DEBUG_${${ICU_PRIVATE_VAR_NS}_COMPONENT}) + endforeach(${ICU_PRIVATE_VAR_NS}_COMPONENT) + + # Try to find out compiler flags + find_program(${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE icu-config HINTS ${${ICU_PRIVATE_VAR_NS}_ROOT}) + if(${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE) + execute_process(COMMAND ${${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE} --cflags OUTPUT_VARIABLE ${ICU_PUBLIC_VAR_NS}_C_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE} --cxxflags OUTPUT_VARIABLE ${ICU_PUBLIC_VAR_NS}_CXX_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE} --cppflags OUTPUT_VARIABLE ${ICU_PUBLIC_VAR_NS}_CPP_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + + execute_process(COMMAND ${${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE} --cflags-dynamic OUTPUT_VARIABLE ${ICU_PUBLIC_VAR_NS}_C_SHARED_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE} --cxxflags-dynamic OUTPUT_VARIABLE ${ICU_PUBLIC_VAR_NS}_CXX_SHARED_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE} --cppflags-dynamic OUTPUT_VARIABLE ${ICU_PUBLIC_VAR_NS}_CPP_SHARED_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + endif(${ICU_PUBLIC_VAR_NS}_CONFIG_EXECUTABLE) + + # Check find_package arguments + include(FindPackageHandleStandardArgs) + if(${ICU_PUBLIC_VAR_NS}_FIND_REQUIRED AND NOT ${ICU_PUBLIC_VAR_NS}_FIND_QUIETLY) + find_package_handle_standard_args( + ${ICU_PUBLIC_VAR_NS} + REQUIRED_VARS ${ICU_PUBLIC_VAR_NS}_LIBRARIES ${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS + VERSION_VAR ${ICU_PUBLIC_VAR_NS}_VERSION + ) + else(${ICU_PUBLIC_VAR_NS}_FIND_REQUIRED AND NOT ${ICU_PUBLIC_VAR_NS}_FIND_QUIETLY) + find_package_handle_standard_args(${ICU_PUBLIC_VAR_NS} "ICU not found" ${ICU_PUBLIC_VAR_NS}_LIBRARIES ${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS) + endif(${ICU_PUBLIC_VAR_NS}_FIND_REQUIRED AND NOT ${ICU_PUBLIC_VAR_NS}_FIND_QUIETLY) +else(${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS) + if(${ICU_PUBLIC_VAR_NS}_FIND_REQUIRED AND NOT ${ICU_PUBLIC_VAR_NS}_FIND_QUIETLY) + message(FATAL_ERROR "Could not find ICU include directory") + endif(${ICU_PUBLIC_VAR_NS}_FIND_REQUIRED AND NOT ${ICU_PUBLIC_VAR_NS}_FIND_QUIETLY) +endif(${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS) + +mark_as_advanced( + ${ICU_PUBLIC_VAR_NS}_INCLUDE_DIRS + ${ICU_PUBLIC_VAR_NS}_LIBRARIES +) + +# IN (args) +icudebug("FIND_COMPONENTS") +icudebug("FIND_REQUIRED") +icudebug("FIND_QUIETLY") +icudebug("FIND_VERSION") +# OUT +# Found +icudebug("FOUND") +icudebug("UC_FOUND") +icudebug("I18N_FOUND") +icudebug("IO_FOUND") +icudebug("LE_FOUND") +icudebug("LX_FOUND") +icudebug("DATA_FOUND") +# Flags +icudebug("C_FLAGS") +icudebug("CPP_FLAGS") +icudebug("CXX_FLAGS") +icudebug("C_SHARED_FLAGS") +icudebug("CPP_SHARED_FLAGS") +icudebug("CXX_SHARED_FLAGS") +# Linking +icudebug("INCLUDE_DIRS") +icudebug("LIBRARIES") +# Version +icudebug("MAJOR_VERSION") +icudebug("MINOR_VERSION") +icudebug("PATCH_VERSION") +icudebug("VERSION") diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000000..671e4d9c6b6c --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,191 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +cmake_minimum_required(VERSION 3.10) +set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) + +# set the project name +project(velox) + +# If CODEGEN support isn't explicitly set, we guestimate the value based on the +# compiler +if((NOT DEFINED CODEGEN_SUPPORT) AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang")) + message(STATUS "Enabling Codegen") + set(CODEGEN_SUPPORT True) +else() + message(STATUS "Disabling Codegen") + set(CODEGEN_SUPPORT False) +endif() + +# define processor variable for conditional compilation +if(${CODEGEN_SUPPORT}) + add_compile_definitions(CODEGEN_ENABLED=1) +endif() + +# MacOSX enables two-level namespace by default: +# http://mirror.informatimago.com/next/developer.apple.com/releasenotes/DeveloperTools/TwoLevelNamespaces.html +# Enables -flat_namespace so type_info can be deudplicated across .so boundaries +if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + add_link_options("-Wl,-flat_namespace") +endif() + +if(UNIX AND NOT APPLE) + # codegen linker flags, -export-dynamic for rtti + add_link_options("-Wl,-export-dynamic") +endif() + +# Required so velox code can be used in a dynamic library +set(POSITION_INDEPENDENT_CODE True) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -mavx2 -mfma -mavx -mf16c -mbmi2 -march=native -D USE_VELOX_COMMON_BASE -g" +) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D HAS_UNCAUGHT_EXCEPTIONS") + +# Under Ninja, we are able to designate certain targets large enough to require +# restricted parallelism. +if("${MAX_HIGH_MEM_JOBS}") + set_property(GLOBAL PROPERTY JOB_POOLS + "high_memory_pool=${MAX_HIGH_MEM_JOBS}") +else() + set_property(GLOBAL PROPERTY JOB_POOLS high_memory_pool=1000) +endif() + +if("${MAX_LINK_JOBS}") + set_property(GLOBAL APPEND PROPERTY JOB_POOLS + "link_job_pool=${MAX_LINK_JOBS}") + set(CMAKE_JOB_POOL_LINK link_job_pool) +endif() + +if("${TREAT_WARNINGS_AS_ERRORS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") +endif() + +if("${ENABLE_ALL_WARNINGS}") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(KNOWN_COMPILER_SPECIFIC_WARNINGS + "-Wno-range-loop-analysis -Wno-mismatched-tags") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(KNOWN_COMPILER_SPECIFIC_WARNINGS + "-Wno-implicit-fallthrough \ + -Wno-empty-body \ + -Wno-class-memaccess \ + -Wno-comment \ + -Wno-int-in-bool-context \ + -Wno-redundant-move \ + -Wno-type-limits") + endif() + + set(KNOWN_WARNINGS + "-Wno-unused \ + -Wno-unused-parameter \ + -Wno-sign-compare \ + -Wno-ignored-qualifiers \ + ${KNOWN_COMPILER_SPECIFIC_WARNINGS}") + + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra ${KNOWN_WARNINGS}") +endif() + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(Boost_USE_MULTITHREADED TRUE) +find_package( + Boost + 1.66.0 + REQUIRED + program_options + context + filesystem + regex + thread + system + date_time + atomic) +include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) + +# Range-v3 will be enable when the codegen code actually lands keeping it here +# for reference. find_package(range-v3) + +find_package(GTest REQUIRED) +find_package(gflags COMPONENTS shared) + +find_library(GMock gmock) + +find_library(GLOG glog) +include_directories(SYSTEM ${GTEST_INCLUDE_DIRS}) + +find_library(FMT fmt) + +find_library(EVENT event) + +find_library(DOUBLE_CONVERSION double-conversion) + +find_library(LZ4 lz4) +find_library(LZO lzo2) +find_library(RE2 re2 REQUIRED) +find_library(ZSTD zstd) +find_package(ZLIB) +find_library(SNAPPY snappy) + +if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + set(CMAKE_PREFIX_PATH "/usr/local/opt/icu4c" ${CMAKE_PREFIX_PATH}) + find_package(ICU REQUIRED) + include_directories(${ICU_INCLUDE_DIRS}) + link_directories("${ICU_INCLUDE_DIRS}/../lib") +endif() + +find_package(folly CONFIG REQUIRED) +set(FOLLY_WITH_DEPENDENCIES + ${FOLLY_LIBRARIES} ${Boost_LIBRARIES} ${DOUBLE_CONVERSION_LIBRARIES} + ${EVENT} ${SNAPPY} ${CMAKE_DL_LIBS}) + +set(FOLLY ${FOLLY_LIBRARIES}) +set(FOLLY_BENCHMARK Folly::follybenchmark) + +find_package(BZip2 MODULE) +if(BZIP2_FOUND) + list(APPEND FOLLY_WITH_DEPENDENCIES ${BZIP2_LIBRARIES}) +endif() + +include_directories(SYSTEM ${FOLLY_INCLUDE_DIRS}) + +find_package(Protobuf REQUIRED) +include_directories(SYSTEM ${Protobuf_INCLUDE_DIRS}) + +# GCC needs to link a library to enable std::filesystem. +if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") + set(FILESYSTEM "stdc++fs") + + # Ensure we have gcc at least 8+. + if(CMAKE_CXX_COMPILER_VERSION LESS 8.0) + message( + FATAL_ERROR "VELOX requires gcc > 8. Found ${CMAKE_CXX_COMPILER_VERSION}") + endif() +else() + set(FILESYSTEM "") +endif() + +include_directories(SYSTEM velox) +include_directories(SYSTEM velox/external) +include_directories(SYSTEM velox/external/duckdb) + +include(CTest) # include after project() but before add_subdirectory() + +include_directories(.) + +# TODO: Include all other installation files. For now just making sure this +# generates an installable makefile. +install(FILES velox/type/Type.h DESTINATION "include/velox") + +add_subdirectory(velox) diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000000..b09cd7856d58 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 000000000000..18a75c779239 --- /dev/null +++ b/Makefile @@ -0,0 +1,90 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +.PHONY: all cmake build clean debug release unit + +BUILD_BASE_DIR=_build +BUILD_DIR=release +BUILD_TYPE=Release +TREAT_WARNINGS_AS_ERRORS ?= 1 +ENABLE_WALL ?= 1 +WARNINGS_AS_ERRORS=-DTREAT_WARNINGS_AS_ERRORS=${TREAT_WARNINGS_AS_ERRORS} +ENABLE_ALL_WARNINGS=-DENABLE_ALL_WARNINGS=${ENABLE_WALL} + +# Use Ninja if available. If Ninja is used, pass through parallelism control flags. +USE_NINJA ?= 1 +ifeq ($(USE_NINJA), 1) +ifneq ($(shell which ninja), ) +GENERATOR=-GNinja -DMAX_LINK_JOBS=$(MAX_LINK_JOBS) -DMAX_HIGH_MEM_JOBS=$(MAX_HIGH_MEM_JOBS) +endif +endif + +ifndef USE_CCACHE +ifneq ($(shell which ccache), ) +USE_CCACHE=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache +endif +endif + +NUM_THREADS ?= $(shell getconf _NPROCESSORS_CONF 2>/dev/null || echo 1) + +all: release #: Build the release version + +clean: #: Delete all build artifacts + rm -rf $(BUILD_BASE_DIR) + +cmake: #: Use CMake to create a Makefile build system + mkdir -p $(BUILD_BASE_DIR)/$(BUILD_DIR) && \ + cmake -B "$(BUILD_BASE_DIR)/$(BUILD_DIR)" $(GENERATOR) $(USE_CCACHE) $(FORCE_COLOR) ${WARNINGS_AS_ERRORS} ${ENABLE_ALL_WARNINGS} -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) + +build: #: Build the software based in BUILD_DIR and BUILD_TYPE variables + cmake --build $(BUILD_BASE_DIR)/$(BUILD_DIR) -j ${NUM_THREADS} + +debug: #: Build with debugging symbols + $(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=Debug + $(MAKE) build BUILD_DIR=debug + +release: #: Build the release version + $(MAKE) cmake BUILD_DIR=release BUILD_TYPE=Release && \ + $(MAKE) build BUILD_DIR=release + +unittest: debug #: Build with debugging and run unit tests + cd $(BUILD_BASE_DIR)/debug && ctest -j ${NUM_THREADS} -VV --output-on-failure --exclude-regex "MemoryMemoryHeaderTest\.getDefaultScopedMemoryPool|MemoryManagerTest\.GlobalMemoryManager" + +format-fix: #: Fix formatting issues in the current branch + scripts/check.py format branch --fix + +format-check: #: Check for formatting issues on the current branch + clang-format --version + scripts/check.py format branch + +header-fix: #: Fix license header issues in the current branch + scripts/check.py header branch --fix + +header-check: #: Check for license header issues on the current branch + scripts/check.py header branch + +circleci-container: #: Build the linux container for CircleCi + $(MAKE) linux-container CONTAINER_NAME=circleci + +check-container: + $(MAKE) linux-container CONTAINER_NAME=check + +linux-container: + rm -rf /tmp/docker && \ + mkdir -p /tmp/docker && \ + cp scripts/setup-$(CONTAINER_NAME).sh scripts/$(CONTAINER_NAME)-container.dockfile /tmp/docker && \ + cd /tmp/docker && \ + docker build --tag "prestocpp/velox-$(CONTAINER_NAME):${USER}-$(shell date +%Y%m%d)" -f $(CONTAINER_NAME)-container.dockfile . + +help: #: Show the help messages + @cat $(firstword $(MAKEFILE_LIST)) | \ + awk '/^[-a-z]+:/' | \ + awk -F: '{ printf("%-20s %s\n", $$1, $$NF) }' diff --git a/README.md b/README.md new file mode 100644 index 000000000000..c73fd75a70d6 --- /dev/null +++ b/README.md @@ -0,0 +1,96 @@ +# velox + +## Build Notes + +### Dependencies +For the current set of dependencies please refer to scripts/setup-macos.sh, scripts/setup-linux.sh + +## Building +Run `make` in the root directory to compile the sources. For development, use +`make debug` to build a non-optimized debug version. Use `make unittest` to build +and run tests. + +### Makefile targets +A reminder of the available Makefile targets can be obtained using `make help` +``` + make help + all Build the release version + clean Delete all build artifacts + cmake Use CMake to create a Makefile build system + build Build the software based in BUILD_DIR and BUILD_TYPE variables + debug Build with debugging symbols + release Build the release version + unittest Build with debugging and run unit tests + format-fix Fix formatting issues in the current branch + format-check Check for formatting issues on the current branch + header-fix Fix license header issues in the current branch + header-check Check for license header issues on the current branch + linux-container Build the CircleCi linux container from scratch + help Show the help messages +``` + +## CircleCi Continuous Integration + +Details are in the [.circleci/REAME.md](.circleci) + +## Code formatting, headers + +### Showing, Fixing and Passing Checks + +Makefile targets exist for showing, fixing and checking formatting, license +headers. These targets are shortcuts for calling +`./scripts/check.py`. + +CircleCi runs `make format-check`, `make header-check` as +part of our continious integration. Pull requests should pass format-check and +header-check without errors before being accepted. + +Formatting issues found on the changed lines in the current commit can be +displayed using `make format-show`. These issues can be fixed by using `make +format-fix`. This will apply formatting changes to changed lines in the +current commit. + +Header issues found on the changed files in the current commit can be displayed +using `make header-show`. These issues can be fixed by using `make +header-fix`. This will apply license header updates to files in the current +commit. + +### Importing code + +Code imported from fbcode might pass `make format-check` as is and without +change. We are using the .clang-format config file that is used in fbcode. + +Use `make header-fix` to apply our open source license to imported code. + +An entire directory tree of files can be formatted and have license headers added +using the `tree` variant of the format.sh commands: +``` + ./scripts/check.py format tree + ./scripts/check.py format tree --fix + + ./scripts/check.py header tree + ./scripts/check.py header tree --fix +``` + +All the available formatting commands can be displayed by using +`./scripts/check.py help`. + +There is not currently a mechanism to *opt out* files or directories from the +checks. When we need one it can be added. + +## Development Env. + +### Setting up on macOs + +See `scripts/setup-macos.sh` + +After running the setup script add the cmake-format bin to your $PATH, maybe +something like this in your ~/.profile: + +``` +export PATH=$HOME/bin:$HOME/Library/Python/3.7/bin:$PATH +``` + +### Setting up on Linux (CentOs 8 or later) + +See `scripts/setup-linux.sh` diff --git a/build/deps/github_hashes/facebook/folly-rev.txt b/build/deps/github_hashes/facebook/folly-rev.txt new file mode 100644 index 000000000000..945702576e2b --- /dev/null +++ b/build/deps/github_hashes/facebook/folly-rev.txt @@ -0,0 +1 @@ +Subproject commit 37e6bbcc067b70ac8745444d4921926df8416508 diff --git a/build/fbcode_builder/.gitignore b/build/fbcode_builder/.gitignore new file mode 100644 index 000000000000..b98f3edfa6f9 --- /dev/null +++ b/build/fbcode_builder/.gitignore @@ -0,0 +1,5 @@ +# Facebook-internal CI builds don't have write permission outside of the +# source tree, so we install all projects into this directory. +/facebook_ci +__pycache__/ +*.pyc diff --git a/build/fbcode_builder/CMake/FBBuildOptions.cmake b/build/fbcode_builder/CMake/FBBuildOptions.cmake new file mode 100644 index 000000000000..dbaa29933a05 --- /dev/null +++ b/build/fbcode_builder/CMake/FBBuildOptions.cmake @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +function (fb_activate_static_library_option) + option(USE_STATIC_DEPS_ON_UNIX + "If enabled, use static dependencies on unix systems. This is generally discouraged." + OFF + ) + # Mark USE_STATIC_DEPS_ON_UNIX as an "advanced" option, since enabling it + # is generally discouraged. + mark_as_advanced(USE_STATIC_DEPS_ON_UNIX) + + if(UNIX AND USE_STATIC_DEPS_ON_UNIX) + SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a" PARENT_SCOPE) + endif() +endfunction() diff --git a/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake b/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake new file mode 100644 index 000000000000..933180189d07 --- /dev/null +++ b/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake @@ -0,0 +1,141 @@ +# +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Helper function for parsing arguments to a CMake function. +# +# This function is very similar to CMake's built-in cmake_parse_arguments() +# function, with some improvements: +# - This function correctly handles empty arguments. (cmake_parse_arguments() +# ignores empty arguments.) +# - If a multi-value argument is specified more than once, the subsequent +# arguments are appended to the original list rather than replacing it. e.g. +# if "SOURCES" is a multi-value argument, and the argument list contains +# "SOURCES a b c SOURCES x y z" then the resulting value for SOURCES will be +# "a;b;c;x;y;z" rather than "x;y;z" +# - This function errors out by default on unrecognized arguments. You can +# pass in an extra "ALLOW_UNPARSED_ARGS" argument to make it behave like +# cmake_parse_arguments(), and return the unparsed arguments in a +# _UNPARSED_ARGUMENTS variable instead. +# +# It does look like cmake_parse_arguments() handled empty arguments correctly +# from CMake 3.0 through 3.3, but it seems like this was probably broken when +# it was turned into a built-in function in CMake 3.4. Here is discussion and +# patches that fixed this behavior prior to CMake 3.0: +# https://cmake.org/pipermail/cmake-developers/2013-November/020607.html +# +# The one downside to this function over the built-in cmake_parse_arguments() +# is that I don't think we can achieve the PARSE_ARGV behavior in a non-builtin +# function, so we can't properly handle arguments that contain ";". CMake will +# treat the ";" characters as list element separators, and treat it as multiple +# separate arguments. +# +function(fb_cmake_parse_args PREFIX OPTIONS ONE_VALUE_ARGS MULTI_VALUE_ARGS ARGS) + foreach(option IN LISTS ARGN) + if ("${option}" STREQUAL "ALLOW_UNPARSED_ARGS") + set(ALLOW_UNPARSED_ARGS TRUE) + else() + message( + FATAL_ERROR + "unknown optional argument for fb_cmake_parse_args(): ${option}" + ) + endif() + endforeach() + + # Define all options as FALSE in the parent scope to start with + foreach(var_name IN LISTS OPTIONS) + set("${PREFIX}_${var_name}" "FALSE" PARENT_SCOPE) + endforeach() + + # TODO: We aren't extremely strict about error checking for one-value + # arguments here. e.g., we don't complain if a one-value argument is + # followed by another option/one-value/multi-value name rather than an + # argument. We also don't complain if a one-value argument is the last + # argument and isn't followed by a value. + + list(APPEND all_args ${ONE_VALUE_ARGS}) + list(APPEND all_args ${MULTI_VALUE_ARGS}) + set(current_variable) + set(unparsed_args) + foreach(arg IN LISTS ARGS) + list(FIND OPTIONS "${arg}" opt_index) + if("${opt_index}" EQUAL -1) + list(FIND all_args "${arg}" arg_index) + if("${arg_index}" EQUAL -1) + # This argument does not match an argument name, + # must be an argument value + if("${current_variable}" STREQUAL "") + list(APPEND unparsed_args "${arg}") + else() + # Ugh, CMake lists have a pretty fundamental flaw: they cannot + # distinguish between an empty list and a list with a single empty + # element. We track our own SEEN_VALUES_arg setting to help + # distinguish this and behave properly here. + if ("${SEEN_${current_variable}}" AND "${${current_variable}}" STREQUAL "") + set("${current_variable}" ";${arg}") + else() + list(APPEND "${current_variable}" "${arg}") + endif() + set("SEEN_${current_variable}" TRUE) + endif() + else() + # We found a single- or multi-value argument name + set(current_variable "VALUES_${arg}") + set("SEEN_${arg}" TRUE) + endif() + else() + # We found an option variable + set("${PREFIX}_${arg}" "TRUE" PARENT_SCOPE) + set(current_variable) + endif() + endforeach() + + foreach(arg_name IN LISTS ONE_VALUE_ARGS) + if(NOT "${SEEN_${arg_name}}") + unset("${PREFIX}_${arg_name}" PARENT_SCOPE) + elseif(NOT "${SEEN_VALUES_${arg_name}}") + # If the argument was seen but a value wasn't specified, error out. + # We require exactly one value to be specified. + message( + FATAL_ERROR "argument ${arg_name} was specified without a value" + ) + else() + list(LENGTH "VALUES_${arg_name}" num_args) + if("${num_args}" EQUAL 0) + # We know an argument was specified and that we called list(APPEND). + # If CMake thinks the list is empty that means there is really a single + # empty element in the list. + set("${PREFIX}_${arg_name}" "" PARENT_SCOPE) + elseif("${num_args}" EQUAL 1) + list(GET "VALUES_${arg_name}" 0 arg_value) + set("${PREFIX}_${arg_name}" "${arg_value}" PARENT_SCOPE) + else() + message( + FATAL_ERROR "too many arguments specified for ${arg_name}: " + "${VALUES_${arg_name}}" + ) + endif() + endif() + endforeach() + + foreach(arg_name IN LISTS MULTI_VALUE_ARGS) + # If this argument name was never seen, then unset the parent scope + if (NOT "${SEEN_${arg_name}}") + unset("${PREFIX}_${arg_name}" PARENT_SCOPE) + else() + # TODO: Our caller still won't be able to distinguish between an empty + # list and a list with a single empty element. We can tell which is + # which, but CMake lists don't make it easy to show this to our caller. + set("${PREFIX}_${arg_name}" "${VALUES_${arg_name}}" PARENT_SCOPE) + endif() + endforeach() + + # By default we fatal out on unparsed arguments, but return them to the + # caller if ALLOW_UNPARSED_ARGS was specified. + if (DEFINED unparsed_args) + if ("${ALLOW_UNPARSED_ARGS}") + set("${PREFIX}_UNPARSED_ARGUMENTS" "${unparsed_args}" PARENT_SCOPE) + else() + message(FATAL_ERROR "unrecognized arguments: ${unparsed_args}") + endif() + endif() +endfunction() diff --git a/build/fbcode_builder/CMake/FBCompilerSettings.cmake b/build/fbcode_builder/CMake/FBCompilerSettings.cmake new file mode 100644 index 000000000000..585c953203c8 --- /dev/null +++ b/build/fbcode_builder/CMake/FBCompilerSettings.cmake @@ -0,0 +1,13 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +# This file applies common compiler settings that are shared across +# a number of Facebook opensource projects. +# Please use caution and your best judgement before making changes +# to these shared compiler settings in order to avoid accidentally +# breaking a build in another project! + +if (WIN32) + include(FBCompilerSettingsMSVC) +else() + include(FBCompilerSettingsUnix) +endif() diff --git a/build/fbcode_builder/CMake/FBCompilerSettingsMSVC.cmake b/build/fbcode_builder/CMake/FBCompilerSettingsMSVC.cmake new file mode 100644 index 000000000000..4efd7e9668f0 --- /dev/null +++ b/build/fbcode_builder/CMake/FBCompilerSettingsMSVC.cmake @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +# This file applies common compiler settings that are shared across +# a number of Facebook opensource projects. +# Please use caution and your best judgement before making changes +# to these shared compiler settings in order to avoid accidentally +# breaking a build in another project! + +add_compile_options( + /wd4250 # 'class1' : inherits 'class2::member' via dominance +) diff --git a/build/fbcode_builder/CMake/FBCompilerSettingsUnix.cmake b/build/fbcode_builder/CMake/FBCompilerSettingsUnix.cmake new file mode 100644 index 000000000000..c26ce78b1d20 --- /dev/null +++ b/build/fbcode_builder/CMake/FBCompilerSettingsUnix.cmake @@ -0,0 +1,9 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +# This file applies common compiler settings that are shared across +# a number of Facebook opensource projects. +# Please use caution and your best judgement before making changes +# to these shared compiler settings in order to avoid accidentally +# breaking a build in another project! + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Wextra -Wno-deprecated -Wno-deprecated-declarations") diff --git a/build/fbcode_builder/CMake/FBPythonBinary.cmake b/build/fbcode_builder/CMake/FBPythonBinary.cmake new file mode 100644 index 000000000000..99c33fb8c953 --- /dev/null +++ b/build/fbcode_builder/CMake/FBPythonBinary.cmake @@ -0,0 +1,697 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +include(FBCMakeParseArgs) + +# +# This file contains helper functions for building self-executing Python +# binaries. +# +# This is somewhat different than typical python installation with +# distutils/pip/virtualenv/etc. We primarily want to build a standalone +# executable, isolated from other Python packages on the system. We don't want +# to install files into the standard library python paths. This is more +# similar to PEX (https://github.com/pantsbuild/pex) and XAR +# (https://github.com/facebookincubator/xar). (In the future it would be nice +# to update this code to also support directly generating XAR files if XAR is +# available.) +# +# We also want to be able to easily define "libraries" of python files that can +# be shared and re-used between these standalone python executables, and can be +# shared across projects in different repositories. This means that we do need +# a way to "install" libraries so that they are visible to CMake builds in +# other repositories, without actually installing them in the standard python +# library paths. +# + +# If the caller has not already found Python, do so now. +# If we fail to find python now we won't fail immediately, but +# add_fb_python_executable() or add_fb_python_library() will fatal out if they +# are used. +if(NOT TARGET Python3::Interpreter) + # CMake 3.12+ ships with a FindPython3.cmake module. Try using it first. + # We find with QUIET here, since otherwise this generates some noisy warnings + # on versions of CMake before 3.12 + if (WIN32) + # On Windows we need both the Intepreter as well as the Development + # libraries. + find_package(Python3 COMPONENTS Interpreter Development QUIET) + else() + find_package(Python3 COMPONENTS Interpreter QUIET) + endif() + if(Python3_Interpreter_FOUND) + message(STATUS "Found Python 3: ${Python3_EXECUTABLE}") + else() + # Try with the FindPythonInterp.cmake module available in older CMake + # versions. Check to see if the caller has already searched for this + # themselves first. + if(NOT PYTHONINTERP_FOUND) + set(Python_ADDITIONAL_VERSIONS 3 3.6 3.5 3.4 3.3 3.2 3.1) + find_package(PythonInterp) + # TODO: On Windows we require the Python libraries as well. + # We currently do not search for them on this code path. + # For now we require building with CMake 3.12+ on Windows, so that the + # FindPython3 code path above is available. + endif() + if(PYTHONINTERP_FOUND) + if("${PYTHON_VERSION_MAJOR}" GREATER_EQUAL 3) + set(Python3_EXECUTABLE "${PYTHON_EXECUTABLE}") + add_custom_target(Python3::Interpreter) + else() + string( + CONCAT FBPY_FIND_PYTHON_ERR + "found Python ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}, " + "but need Python 3" + ) + endif() + endif() + endif() +endif() + +# Find our helper program. +# We typically install this in the same directory as this .cmake file. +find_program( + FB_MAKE_PYTHON_ARCHIVE "make_fbpy_archive.py" + PATHS ${CMAKE_MODULE_PATH} +) +set(FB_PY_TEST_MAIN "${CMAKE_CURRENT_LIST_DIR}/fb_py_test_main.py") +set( + FB_PY_TEST_DISCOVER_SCRIPT + "${CMAKE_CURRENT_LIST_DIR}/FBPythonTestAddTests.cmake" +) +set( + FB_PY_WIN_MAIN_C + "${CMAKE_CURRENT_LIST_DIR}/fb_py_win_main.c" +) + +# An option to control the default installation location for +# install_fb_python_library(). This is relative to ${CMAKE_INSTALL_PREFIX} +set( + FBPY_LIB_INSTALL_DIR "lib/fb-py-libs" CACHE STRING + "The subdirectory where FB python libraries should be installed" +) + +# +# Build a self-executing python binary. +# +# This accepts the same arguments as add_fb_python_library(). +# +# In addition, a MAIN_MODULE argument is accepted. This argument specifies +# which module should be started as the __main__ module when the executable is +# run. If left unspecified, a __main__.py script must be present in the +# manifest. +# +function(add_fb_python_executable TARGET) + fb_py_check_available() + + # Parse the arguments + set(one_value_args BASE_DIR NAMESPACE MAIN_MODULE TYPE) + set(multi_value_args SOURCES DEPENDS) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR) + + # Use add_fb_python_library() to perform most of our source handling + add_fb_python_library( + "${TARGET}.main_lib" + BASE_DIR "${ARG_BASE_DIR}" + NAMESPACE "${ARG_NAMESPACE}" + SOURCES ${ARG_SOURCES} + DEPENDS ${ARG_DEPENDS} + ) + + set( + manifest_files + "$" + ) + set( + source_files + "$" + ) + + # The command to build the executable archive. + # + # If we are using CMake 3.8+ we can use COMMAND_EXPAND_LISTS. + # CMP0067 isn't really the policy we care about, but seems like the best way + # to check if we are running 3.8+. + if (POLICY CMP0067) + set(extra_cmd_params COMMAND_EXPAND_LISTS) + set(make_py_args "${manifest_files}") + else() + set(extra_cmd_params) + set(make_py_args --manifest-separator "::" "$") + endif() + + set(output_file "${TARGET}${CMAKE_EXECUTABLE_SUFFIX}") + if(WIN32) + set(zipapp_output "${TARGET}.py_zipapp") + else() + set(zipapp_output "${output_file}") + endif() + set(zipapp_output_file "${zipapp_output}") + + set(is_dir_output FALSE) + if(DEFINED ARG_TYPE) + list(APPEND make_py_args "--type" "${ARG_TYPE}") + if ("${ARG_TYPE}" STREQUAL "dir") + set(is_dir_output TRUE) + # CMake doesn't really seem to like having a directory specified as an + # output; specify the __main__.py file as the output instead. + set(zipapp_output_file "${zipapp_output}/__main__.py") + list(APPEND + extra_cmd_params + COMMAND "${CMAKE_COMMAND}" -E remove_directory "${zipapp_output}" + ) + endif() + endif() + + if(DEFINED ARG_MAIN_MODULE) + list(APPEND make_py_args "--main" "${ARG_MAIN_MODULE}") + endif() + + add_custom_command( + OUTPUT "${zipapp_output_file}" + ${extra_cmd_params} + COMMAND + "${Python3_EXECUTABLE}" "${FB_MAKE_PYTHON_ARCHIVE}" + -o "${zipapp_output}" + ${make_py_args} + DEPENDS + ${source_files} + "${TARGET}.main_lib.py_sources_built" + "${FB_MAKE_PYTHON_ARCHIVE}" + ) + + if(WIN32) + if(is_dir_output) + # TODO: generate a main executable that will invoke Python3 + # with the correct main module inside the output directory + else() + add_executable("${TARGET}.winmain" "${FB_PY_WIN_MAIN_C}") + target_link_libraries("${TARGET}.winmain" Python3::Python) + # The Python3::Python target doesn't seem to be set up completely + # correctly on Windows for some reason, and we have to explicitly add + # ${Python3_LIBRARY_DIRS} to the target link directories. + target_link_directories( + "${TARGET}.winmain" + PUBLIC ${Python3_LIBRARY_DIRS} + ) + add_custom_command( + OUTPUT "${output_file}" + DEPENDS "${TARGET}.winmain" "${zipapp_output_file}" + COMMAND + "cmd.exe" "/c" "copy" "/b" + "${TARGET}.winmain${CMAKE_EXECUTABLE_SUFFIX}+${zipapp_output}" + "${output_file}" + ) + endif() + endif() + + # Add an "ALL" target that depends on force ${TARGET}, + # so that ${TARGET} will be included in the default list of build targets. + add_custom_target("${TARGET}.GEN_PY_EXE" ALL DEPENDS "${output_file}") + + # Allow resolving the executable path for the target that we generate + # via a generator expression like: + # "WATCHMAN_WAIT_PATH=$" + set_property(TARGET "${TARGET}.GEN_PY_EXE" + PROPERTY EXECUTABLE "${CMAKE_CURRENT_BINARY_DIR}/${output_file}") +endfunction() + +# Define a python unittest executable. +# The executable is built using add_fb_python_executable and has the +# following differences: +# +# Each of the source files specified in SOURCES will be imported +# and have unittest discovery performed upon them. +# Those sources will be imported in the top level namespace. +# +# The ENV argument allows specifying a list of "KEY=VALUE" +# pairs that will be used by the test runner to set up the environment +# in the child process prior to running the test. This is useful for +# passing additional configuration to the test. +function(add_fb_python_unittest TARGET) + # Parse the arguments + set(multi_value_args SOURCES DEPENDS ENV PROPERTIES) + set( + one_value_args + WORKING_DIRECTORY BASE_DIR NAMESPACE TEST_LIST DISCOVERY_TIMEOUT + ) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR) + if(NOT ARG_WORKING_DIRECTORY) + # Default the working directory to the current binary directory. + # This matches the default behavior of add_test() and other standard + # test functions like gtest_discover_tests() + set(ARG_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + endif() + if(NOT ARG_TEST_LIST) + set(ARG_TEST_LIST "${TARGET}_TESTS") + endif() + if(NOT ARG_DISCOVERY_TIMEOUT) + set(ARG_DISCOVERY_TIMEOUT 5) + endif() + + # Tell our test program the list of modules to scan for tests. + # We scan all modules directly listed in our SOURCES argument, and skip + # modules that came from dependencies in the DEPENDS list. + # + # This is written into a __test_modules__.py module that the test runner + # will look at. + set( + test_modules_path + "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_test_modules.py" + ) + file(WRITE "${test_modules_path}" "TEST_MODULES = [\n") + string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}") + if (NOT "${namespace_dir}" STREQUAL "") + set(namespace_dir "${namespace_dir}/") + endif() + set(test_modules) + foreach(src_path IN LISTS ARG_SOURCES) + fb_py_compute_dest_path( + abs_source dest_path + "${src_path}" "${namespace_dir}" "${ARG_BASE_DIR}" + ) + string(REPLACE "/" "." module_name "${dest_path}") + string(REGEX REPLACE "\\.py$" "" module_name "${module_name}") + list(APPEND test_modules "${module_name}") + file(APPEND "${test_modules_path}" " '${module_name}',\n") + endforeach() + file(APPEND "${test_modules_path}" "]\n") + + # The __main__ is provided by our runner wrapper/bootstrap + list(APPEND ARG_SOURCES "${FB_PY_TEST_MAIN}=__main__.py") + list(APPEND ARG_SOURCES "${test_modules_path}=__test_modules__.py") + + add_fb_python_executable( + "${TARGET}" + NAMESPACE "${ARG_NAMESPACE}" + BASE_DIR "${ARG_BASE_DIR}" + SOURCES ${ARG_SOURCES} + DEPENDS ${ARG_DEPENDS} + ) + + # Run test discovery after the test executable is built. + # This logic is based on the code for gtest_discover_tests() + set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}") + set(ctest_include_file "${ctest_file_base}_include.cmake") + set(ctest_tests_file "${ctest_file_base}_tests.cmake") + add_custom_command( + TARGET "${TARGET}.GEN_PY_EXE" POST_BUILD + BYPRODUCTS "${ctest_tests_file}" + COMMAND + "${CMAKE_COMMAND}" + -D "TEST_TARGET=${TARGET}" + -D "TEST_INTERPRETER=${Python3_EXECUTABLE}" + -D "TEST_ENV=${ARG_ENV}" + -D "TEST_EXECUTABLE=$" + -D "TEST_WORKING_DIR=${ARG_WORKING_DIRECTORY}" + -D "TEST_LIST=${ARG_TEST_LIST}" + -D "TEST_PREFIX=${TARGET}::" + -D "TEST_PROPERTIES=${ARG_PROPERTIES}" + -D "CTEST_FILE=${ctest_tests_file}" + -P "${FB_PY_TEST_DISCOVER_SCRIPT}" + VERBATIM + ) + + file( + WRITE "${ctest_include_file}" + "if(EXISTS \"${ctest_tests_file}\")\n" + " include(\"${ctest_tests_file}\")\n" + "else()\n" + " add_test(\"${TARGET}_NOT_BUILT\" \"${TARGET}_NOT_BUILT\")\n" + "endif()\n" + ) + set_property( + DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES + "${ctest_include_file}" + ) +endfunction() + +# +# Define a python library. +# +# If you want to install a python library generated from this rule note that +# you need to use install_fb_python_library() rather than CMake's built-in +# install() function. This will make it available for other downstream +# projects to use in their add_fb_python_executable() and +# add_fb_python_library() calls. (You do still need to use `install(EXPORT)` +# later to install the CMake exports.) +# +# Parameters: +# - BASE_DIR : +# The base directory path to strip off from each source path. All source +# files must be inside this directory. If not specified it defaults to +# ${CMAKE_CURRENT_SOURCE_DIR}. +# - NAMESPACE : +# The destination namespace where these files should be installed in python +# binaries. If not specified, this defaults to the current relative path of +# ${CMAKE_CURRENT_SOURCE_DIR} inside ${CMAKE_SOURCE_DIR}. e.g., a python +# library defined in the directory repo_root/foo/bar will use a default +# namespace of "foo.bar" +# - SOURCES <...>: +# The python source files. +# You may optionally specify as source using the form: PATH=ALIAS where +# PATH is a relative path in the source tree and ALIAS is the relative +# path into which PATH should be rewritten. This is useful for mapping +# an executable script to the main module in a python executable. +# e.g.: `python/bin/watchman-wait=__main__.py` +# - DEPENDS <...>: +# Other python libraries that this one depends on. +# - INSTALL_DIR : +# The directory where this library should be installed. +# install_fb_python_library() must still be called later to perform the +# installation. If a relative path is given it will be treated relative to +# ${CMAKE_INSTALL_PREFIX} +# +# CMake is unfortunately pretty crappy at being able to define custom build +# rules & behaviors. It doesn't support transitive property propagation +# between custom targets; only the built-in add_executable() and add_library() +# targets support transitive properties. +# +# We hack around this janky CMake behavior by (ab)using interface libraries to +# propagate some of the data we want between targets, without actually +# generating a C library. +# +# add_fb_python_library(SOMELIB) generates the following things: +# - An INTERFACE library rule named SOMELIB.py_lib which tracks some +# information about transitive dependencies: +# - the transitive set of source files in the INTERFACE_SOURCES property +# - the transitive set of manifest files that this library depends on in +# the INTERFACE_INCLUDE_DIRECTORIES property. +# - A custom command that generates a SOMELIB.manifest file. +# This file contains the mapping of source files to desired destination +# locations in executables that depend on this library. This manifest file +# will then be read at build-time in order to build executables. +# +function(add_fb_python_library LIB_NAME) + fb_py_check_available() + + # Parse the arguments + # We use fb_cmake_parse_args() rather than cmake_parse_arguments() since + # cmake_parse_arguments() does not handle empty arguments, and it is common + # for callers to want to specify an empty NAMESPACE parameter. + set(one_value_args BASE_DIR NAMESPACE INSTALL_DIR) + set(multi_value_args SOURCES DEPENDS) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR) + + string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}") + if (NOT "${namespace_dir}" STREQUAL "") + set(namespace_dir "${namespace_dir}/") + endif() + + if(NOT DEFINED ARG_INSTALL_DIR) + set(install_dir "${FBPY_LIB_INSTALL_DIR}/") + elseif("${ARG_INSTALL_DIR}" STREQUAL "") + set(install_dir "") + else() + set(install_dir "${ARG_INSTALL_DIR}/") + endif() + + # message(STATUS "fb py library ${LIB_NAME}: " + # "NS=${namespace_dir} BASE=${ARG_BASE_DIR}") + + # TODO: In the future it would be nice to support pre-compiling the source + # files. We could emit a rule to compile each source file and emit a + # .pyc/.pyo file here, and then have the manifest reference the pyc/pyo + # files. + + # Define a library target to help pass around information about the library, + # and propagate dependency information. + # + # CMake make a lot of assumptions that libraries are C++ libraries. To help + # avoid confusion we name our target "${LIB_NAME}.py_lib" rather than just + # "${LIB_NAME}". This helps avoid confusion if callers try to use + # "${LIB_NAME}" on their own as a target name. (e.g., attempting to install + # it directly with install(TARGETS) won't work. Callers must use + # install_fb_python_library() instead.) + add_library("${LIB_NAME}.py_lib" INTERFACE) + + # Emit the manifest file. + # + # We write the manifest file to a temporary path first, then copy it with + # configure_file(COPYONLY). This is necessary to get CMake to understand + # that "${manifest_path}" is generated by the CMake configure phase, + # and allow using it as a dependency for add_custom_command(). + # (https://gitlab.kitware.com/cmake/cmake/issues/16367) + set(manifest_path "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.manifest") + set(tmp_manifest "${manifest_path}.tmp") + file(WRITE "${tmp_manifest}" "FBPY_MANIFEST 1\n") + set(abs_sources) + foreach(src_path IN LISTS ARG_SOURCES) + fb_py_compute_dest_path( + abs_source dest_path + "${src_path}" "${namespace_dir}" "${ARG_BASE_DIR}" + ) + list(APPEND abs_sources "${abs_source}") + target_sources( + "${LIB_NAME}.py_lib" INTERFACE + "$" + "$" + ) + file( + APPEND "${tmp_manifest}" + "${abs_source} :: ${dest_path}\n" + ) + endforeach() + configure_file("${tmp_manifest}" "${manifest_path}" COPYONLY) + + target_include_directories( + "${LIB_NAME}.py_lib" INTERFACE + "$" + "$" + ) + + # Add a target that depends on all of the source files. + # This is needed in case some of the source files are generated. This will + # ensure that these source files are brought up-to-date before we build + # any python binaries that depend on this library. + add_custom_target("${LIB_NAME}.py_sources_built" DEPENDS ${abs_sources}) + add_dependencies("${LIB_NAME}.py_lib" "${LIB_NAME}.py_sources_built") + + # Hook up library dependencies, and also make the *.py_sources_built target + # depend on the sources for all of our dependencies also being up-to-date. + foreach(dep IN LISTS ARG_DEPENDS) + target_link_libraries("${LIB_NAME}.py_lib" INTERFACE "${dep}.py_lib") + + # Mark that our .py_sources_built target depends on each our our dependent + # libraries. This serves two functions: + # - This causes CMake to generate an error message if one of the + # dependencies is never defined. The target_link_libraries() call above + # won't complain if one of the dependencies doesn't exist (since it is + # intended to allow passing in file names for plain library files rather + # than just targets). + # - It ensures that sources for our depencencies are built before any + # executable that depends on us. Note that we depend on "${dep}.py_lib" + # rather than "${dep}.py_sources_built" for this purpose because the + # ".py_sources_built" target won't be available for imported targets. + add_dependencies("${LIB_NAME}.py_sources_built" "${dep}.py_lib") + endforeach() + + # Add a custom command to help with library installation, in case + # install_fb_python_library() is called later for this library. + # add_custom_command() only works with file dependencies defined in the same + # CMakeLists.txt file, so we want to make sure this is defined here, rather + # then where install_fb_python_library() is called. + # This command won't be run by default, but will only be run if it is needed + # by a subsequent install_fb_python_library() call. + # + # This command copies the library contents into the build directory. + # It would be nicer if we could skip this intermediate copy, and just run + # make_fbpy_archive.py at install time to copy them directly to the desired + # installation directory. Unfortunately this is difficult to do, and seems + # to interfere with some of the CMake code that wants to generate a manifest + # of installed files. + set(build_install_dir "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.lib_install") + add_custom_command( + OUTPUT + "${build_install_dir}/${LIB_NAME}.manifest" + COMMAND "${CMAKE_COMMAND}" -E remove_directory "${build_install_dir}" + COMMAND + "${Python3_EXECUTABLE}" "${FB_MAKE_PYTHON_ARCHIVE}" --type lib-install + --install-dir "${LIB_NAME}" + -o "${build_install_dir}/${LIB_NAME}" "${manifest_path}" + DEPENDS + "${abs_sources}" + "${manifest_path}" + "${FB_MAKE_PYTHON_ARCHIVE}" + ) + add_custom_target( + "${LIB_NAME}.py_lib_install" + DEPENDS "${build_install_dir}/${LIB_NAME}.manifest" + ) + + # Set some properties to pass through the install paths to + # install_fb_python_library() + # + # Passing through ${build_install_dir} allows install_fb_python_library() + # to work even if used from a different CMakeLists.txt file than where + # add_fb_python_library() was called (i.e. such that + # ${CMAKE_CURRENT_BINARY_DIR} is different between the two calls). + set(abs_install_dir "${install_dir}") + if(NOT IS_ABSOLUTE "${abs_install_dir}") + set(abs_install_dir "${CMAKE_INSTALL_PREFIX}/${abs_install_dir}") + endif() + string(REGEX REPLACE "/$" "" abs_install_dir "${abs_install_dir}") + set_target_properties( + "${LIB_NAME}.py_lib_install" + PROPERTIES + INSTALL_DIR "${abs_install_dir}" + BUILD_INSTALL_DIR "${build_install_dir}" + ) +endfunction() + +# +# Install an FB-style packaged python binary. +# +# - DESTINATION : +# Associate the installed target files with the given export-name. +# +function(install_fb_python_executable TARGET) + # Parse the arguments + set(one_value_args DESTINATION) + set(multi_value_args) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + + if(NOT DEFINED ARG_DESTINATION) + set(ARG_DESTINATION bin) + endif() + + install( + PROGRAMS "$" + DESTINATION "${ARG_DESTINATION}" + ) +endfunction() + +# +# Install a python library. +# +# - EXPORT : +# Associate the installed target files with the given export-name. +# +# Note that unlike the built-in CMake install() function we do not accept a +# DESTINATION parameter. Instead, use the INSTALL_DIR parameter to +# add_fb_python_library() to set the installation location. +# +function(install_fb_python_library LIB_NAME) + set(one_value_args EXPORT) + fb_cmake_parse_args(ARG "" "${one_value_args}" "" "${ARGN}") + + # Export our "${LIB_NAME}.py_lib" target so that it will be available to + # downstream projects in our installed CMake config files. + if(DEFINED ARG_EXPORT) + install(TARGETS "${LIB_NAME}.py_lib" EXPORT "${ARG_EXPORT}") + endif() + + # add_fb_python_library() emits a .py_lib_install target that will prepare + # the installation directory. However, it isn't part of the "ALL" target and + # therefore isn't built by default. + # + # Make sure the ALL target depends on it now. We have to do this by + # introducing yet another custom target. + # Add it as a dependency to the ALL target now. + add_custom_target("${LIB_NAME}.py_lib_install_all" ALL) + add_dependencies( + "${LIB_NAME}.py_lib_install_all" "${LIB_NAME}.py_lib_install" + ) + + # Copy the intermediate install directory generated at build time into + # the desired install location. + get_target_property(dest_dir "${LIB_NAME}.py_lib_install" "INSTALL_DIR") + get_target_property( + build_install_dir "${LIB_NAME}.py_lib_install" "BUILD_INSTALL_DIR" + ) + install( + DIRECTORY "${build_install_dir}/${LIB_NAME}" + DESTINATION "${dest_dir}" + ) + install( + FILES "${build_install_dir}/${LIB_NAME}.manifest" + DESTINATION "${dest_dir}" + ) +endfunction() + +# Helper macro to process the BASE_DIR and NAMESPACE arguments for +# add_fb_python_executable() and add_fb_python_executable() +macro(fb_py_process_default_args NAMESPACE_VAR BASE_DIR_VAR) + # If the namespace was not specified, default to the relative path to the + # current directory (starting from the repository root). + if(NOT DEFINED "${NAMESPACE_VAR}") + file( + RELATIVE_PATH "${NAMESPACE_VAR}" + "${CMAKE_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" + ) + endif() + + if(NOT DEFINED "${BASE_DIR_VAR}") + # If the base directory was not specified, default to the current directory + set("${BASE_DIR_VAR}" "${CMAKE_CURRENT_SOURCE_DIR}") + else() + # If the base directory was specified, always convert it to an + # absolute path. + get_filename_component("${BASE_DIR_VAR}" "${${BASE_DIR_VAR}}" ABSOLUTE) + endif() +endmacro() + +function(fb_py_check_available) + # Make sure that Python 3 and our make_fbpy_archive.py helper script are + # available. + if(NOT Python3_EXECUTABLE) + if(FBPY_FIND_PYTHON_ERR) + message(FATAL_ERROR "Unable to find Python 3: ${FBPY_FIND_PYTHON_ERR}") + else() + message(FATAL_ERROR "Unable to find Python 3") + endif() + endif() + + if (NOT FB_MAKE_PYTHON_ARCHIVE) + message( + FATAL_ERROR "unable to find make_fbpy_archive.py helper program (it " + "should be located in the same directory as FBPythonBinary.cmake)" + ) + endif() +endfunction() + +function( + fb_py_compute_dest_path + src_path_output dest_path_output src_path namespace_dir base_dir +) + if("${src_path}" MATCHES "=") + # We want to split the string on the `=` sign, but cmake doesn't + # provide much in the way of helpers for this, so we rewrite the + # `=` sign to `;` so that we can treat it as a cmake list and + # then index into the components + string(REPLACE "=" ";" src_path_list "${src_path}") + list(GET src_path_list 0 src_path) + # Note that we ignore the `namespace_dir` in the alias case + # in order to allow aliasing a source to the top level `__main__.py` + # filename. + list(GET src_path_list 1 dest_path) + else() + unset(dest_path) + endif() + + get_filename_component(abs_source "${src_path}" ABSOLUTE) + if(NOT DEFINED dest_path) + file(RELATIVE_PATH rel_src "${ARG_BASE_DIR}" "${abs_source}") + if("${rel_src}" MATCHES "^../") + message( + FATAL_ERROR "${LIB_NAME}: source file \"${abs_source}\" is not inside " + "the base directory ${ARG_BASE_DIR}" + ) + endif() + set(dest_path "${namespace_dir}${rel_src}") + endif() + + set("${src_path_output}" "${abs_source}" PARENT_SCOPE) + set("${dest_path_output}" "${dest_path}" PARENT_SCOPE) +endfunction() diff --git a/build/fbcode_builder/CMake/FBPythonTestAddTests.cmake b/build/fbcode_builder/CMake/FBPythonTestAddTests.cmake new file mode 100644 index 000000000000..d73c055d8245 --- /dev/null +++ b/build/fbcode_builder/CMake/FBPythonTestAddTests.cmake @@ -0,0 +1,59 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +# Add a command to be emitted to the CTest file +set(ctest_script) +function(add_command CMD) + set(escaped_args "") + foreach(arg ${ARGN}) + # Escape all arguments using "Bracket Argument" syntax + # We could skip this for argument that don't contain any special + # characters if we wanted to make the output slightly more human-friendly. + set(escaped_args "${escaped_args} [==[${arg}]==]") + endforeach() + set(ctest_script "${ctest_script}${CMD}(${escaped_args})\n" PARENT_SCOPE) +endfunction() + +if(NOT EXISTS "${TEST_EXECUTABLE}") + message(FATAL_ERROR "Test executable does not exist: ${TEST_EXECUTABLE}") +endif() +execute_process( + COMMAND ${CMAKE_COMMAND} -E env ${TEST_ENV} "${TEST_INTERPRETER}" "${TEST_EXECUTABLE}" --list-tests + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + OUTPUT_VARIABLE output + RESULT_VARIABLE result +) +if(NOT "${result}" EQUAL 0) + string(REPLACE "\n" "\n " output "${output}") + message( + FATAL_ERROR + "Error running test executable: ${TEST_EXECUTABLE}\n" + "Output:\n" + " ${output}\n" + ) +endif() + +# Parse output +string(REPLACE "\n" ";" tests_list "${output}") +foreach(test_name ${tests_list}) + add_command( + add_test + "${TEST_PREFIX}${test_name}" + ${CMAKE_COMMAND} -E env ${TEST_ENV} + "${TEST_INTERPRETER}" "${TEST_EXECUTABLE}" "${test_name}" + ) + add_command( + set_tests_properties + "${TEST_PREFIX}${test_name}" + PROPERTIES + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + ${TEST_PROPERTIES} + ) +endforeach() + +# Set a list of discovered tests in the parent scope, in case users +# want access to this list as a CMake variable +if(TEST_LIST) + add_command(set ${TEST_LIST} ${tests_list}) +endif() + +file(WRITE "${CTEST_FILE}" "${ctest_script}") diff --git a/build/fbcode_builder/CMake/FBThriftCppLibrary.cmake b/build/fbcode_builder/CMake/FBThriftCppLibrary.cmake new file mode 100644 index 000000000000..670771a4605a --- /dev/null +++ b/build/fbcode_builder/CMake/FBThriftCppLibrary.cmake @@ -0,0 +1,194 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +include(FBCMakeParseArgs) + +# Generate a C++ library from a thrift file +# +# Parameters: +# - SERVICES [ ...] +# The names of the services defined in the thrift file. +# - DEPENDS [ ...] +# A list of other thrift C++ libraries that this library depends on. +# - OPTIONS [ ...] +# A list of options to pass to the thrift compiler. +# - INCLUDE_DIR +# The sub-directory where generated headers will be installed. +# Defaults to "include" if not specified. The caller must still call +# install() to install the thrift library if desired. +# - THRIFT_INCLUDE_DIR +# The sub-directory where generated headers will be installed. +# Defaults to "${INCLUDE_DIR}/thrift-files" if not specified. +# The caller must still call install() to install the thrift library if +# desired. +function(add_fbthrift_cpp_library LIB_NAME THRIFT_FILE) + # Parse the arguments + set(one_value_args INCLUDE_DIR THRIFT_INCLUDE_DIR) + set(multi_value_args SERVICES DEPENDS OPTIONS) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + if(NOT DEFINED ARG_INCLUDE_DIR) + set(ARG_INCLUDE_DIR "include") + endif() + if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR) + set(ARG_THRIFT_INCLUDE_DIR "${ARG_INCLUDE_DIR}/thrift-files") + endif() + + get_filename_component(base ${THRIFT_FILE} NAME_WE) + get_filename_component( + output_dir + ${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE} + DIRECTORY + ) + + # Generate relative paths in #includes + file( + RELATIVE_PATH include_prefix + "${CMAKE_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}" + ) + get_filename_component(include_prefix ${include_prefix} DIRECTORY) + + if (NOT "${include_prefix}" STREQUAL "") + list(APPEND ARG_OPTIONS "include_prefix=${include_prefix}") + endif() + # CMake 3.12 is finally getting a list(JOIN) function, but until then + # treating the list as a string and replacing the semicolons is good enough. + string(REPLACE ";" "," GEN_ARG_STR "${ARG_OPTIONS}") + + # Compute the list of generated files + list(APPEND generated_headers + "${output_dir}/gen-cpp2/${base}_constants.h" + "${output_dir}/gen-cpp2/${base}_types.h" + "${output_dir}/gen-cpp2/${base}_types.tcc" + "${output_dir}/gen-cpp2/${base}_types_custom_protocol.h" + "${output_dir}/gen-cpp2/${base}_metadata.h" + ) + list(APPEND generated_sources + "${output_dir}/gen-cpp2/${base}_constants.cpp" + "${output_dir}/gen-cpp2/${base}_data.h" + "${output_dir}/gen-cpp2/${base}_data.cpp" + "${output_dir}/gen-cpp2/${base}_types.cpp" + "${output_dir}/gen-cpp2/${base}_metadata.cpp" + ) + foreach(service IN LISTS ARG_SERVICES) + list(APPEND generated_headers + "${output_dir}/gen-cpp2/${service}.h" + "${output_dir}/gen-cpp2/${service}.tcc" + "${output_dir}/gen-cpp2/${service}AsyncClient.h" + "${output_dir}/gen-cpp2/${service}_custom_protocol.h" + ) + list(APPEND generated_sources + "${output_dir}/gen-cpp2/${service}.cpp" + "${output_dir}/gen-cpp2/${service}AsyncClient.cpp" + "${output_dir}/gen-cpp2/${service}_processmap_binary.cpp" + "${output_dir}/gen-cpp2/${service}_processmap_compact.cpp" + ) + endforeach() + + # This generator expression gets the list of include directories required + # for all of our dependencies. + # It requires using COMMAND_EXPAND_LISTS in the add_custom_command() call + # below. COMMAND_EXPAND_LISTS is only available in CMake 3.8+ + # If we really had to support older versions of CMake we would probably need + # to use a wrapper script around the thrift compiler that could take the + # include list as a single argument and split it up before invoking the + # thrift compiler. + if (NOT POLICY CMP0067) + message(FATAL_ERROR "add_fbthrift_cpp_library() requires CMake 3.8+") + endif() + set( + thrift_include_options + "-I;$,;-I;>" + ) + + # Emit the rule to run the thrift compiler + add_custom_command( + OUTPUT + ${generated_headers} + ${generated_sources} + COMMAND_EXPAND_LISTS + COMMAND + "${CMAKE_COMMAND}" -E make_directory "${output_dir}" + COMMAND + "${FBTHRIFT_COMPILER}" + --strict + --gen "mstch_cpp2:${GEN_ARG_STR}" + "${thrift_include_options}" + -o "${output_dir}" + "${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}" + WORKING_DIRECTORY + "${CMAKE_BINARY_DIR}" + MAIN_DEPENDENCY + "${THRIFT_FILE}" + DEPENDS + ${ARG_DEPENDS} + "${FBTHRIFT_COMPILER}" + ) + + # Now emit the library rule to compile the sources + if (BUILD_SHARED_LIBS) + set(LIB_TYPE SHARED) + else () + set(LIB_TYPE STATIC) + endif () + + add_library( + "${LIB_NAME}" ${LIB_TYPE} + ${generated_sources} + ) + + target_include_directories( + "${LIB_NAME}" + PUBLIC + "$" + "$" + ) + target_link_libraries( + "${LIB_NAME}" + PUBLIC + ${ARG_DEPENDS} + FBThrift::thriftcpp2 + Folly::folly + ) + + # Add ${generated_headers} to the PUBLIC_HEADER property for ${LIB_NAME} + # + # This allows callers to install it using + # "install(TARGETS ${LIB_NAME} PUBLIC_HEADER)" + # However, note that CMake's PUBLIC_HEADER behavior is rather inflexible, + # and does have any way to preserve header directory structure. Callers + # must be careful to use the correct PUBLIC_HEADER DESTINATION parameter + # when doing this, to put the files the correct directory themselves. + # We define a HEADER_INSTALL_DIR property with the include directory prefix, + # so typically callers should specify the PUBLIC_HEADER DESTINATION as + # "$" + set_property( + TARGET "${LIB_NAME}" + PROPERTY PUBLIC_HEADER ${generated_headers} + ) + + # Define a dummy interface library to help propagate the thrift include + # directories between dependencies. + add_library("${LIB_NAME}.thrift_includes" INTERFACE) + target_include_directories( + "${LIB_NAME}.thrift_includes" + INTERFACE + "$" + "$" + ) + foreach(dep IN LISTS ARG_DEPENDS) + target_link_libraries( + "${LIB_NAME}.thrift_includes" + INTERFACE "${dep}.thrift_includes" + ) + endforeach() + + set_target_properties( + "${LIB_NAME}" + PROPERTIES + EXPORT_PROPERTIES "THRIFT_INSTALL_DIR" + THRIFT_INSTALL_DIR "${ARG_THRIFT_INCLUDE_DIR}/${include_prefix}" + HEADER_INSTALL_DIR "${ARG_INCLUDE_DIR}/${include_prefix}/gen-cpp2" + ) +endfunction() diff --git a/build/fbcode_builder/CMake/FBThriftLibrary.cmake b/build/fbcode_builder/CMake/FBThriftLibrary.cmake new file mode 100644 index 000000000000..e4280e2a4092 --- /dev/null +++ b/build/fbcode_builder/CMake/FBThriftLibrary.cmake @@ -0,0 +1,77 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +include(FBCMakeParseArgs) +include(FBThriftPyLibrary) +include(FBThriftCppLibrary) + +# +# add_fbthrift_library() +# +# This is a convenience function that generates thrift libraries for multiple +# languages. +# +# For example: +# add_fbthrift_library( +# foo foo.thrift +# LANGUAGES cpp py +# SERVICES Foo +# DEPENDS bar) +# +# will be expanded into two separate calls: +# +# add_fbthrift_cpp_library(foo_cpp foo.thrift SERVICES Foo DEPENDS bar_cpp) +# add_fbthrift_py_library(foo_py foo.thrift SERVICES Foo DEPENDS bar_py) +# +function(add_fbthrift_library LIB_NAME THRIFT_FILE) + # Parse the arguments + set(one_value_args PY_NAMESPACE INCLUDE_DIR THRIFT_INCLUDE_DIR) + set(multi_value_args SERVICES DEPENDS LANGUAGES CPP_OPTIONS PY_OPTIONS) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + + if(NOT DEFINED ARG_INCLUDE_DIR) + set(ARG_INCLUDE_DIR "include") + endif() + if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR) + set(ARG_THRIFT_INCLUDE_DIR "${ARG_INCLUDE_DIR}/thrift-files") + endif() + + # CMake 3.12+ adds list(TRANSFORM) which would be nice to use here, but for + # now we still want to support older versions of CMake. + set(CPP_DEPENDS) + set(PY_DEPENDS) + foreach(dep IN LISTS ARG_DEPENDS) + list(APPEND CPP_DEPENDS "${dep}_cpp") + list(APPEND PY_DEPENDS "${dep}_py") + endforeach() + + foreach(lang IN LISTS ARG_LANGUAGES) + if ("${lang}" STREQUAL "cpp") + add_fbthrift_cpp_library( + "${LIB_NAME}_cpp" "${THRIFT_FILE}" + SERVICES ${ARG_SERVICES} + DEPENDS ${CPP_DEPENDS} + OPTIONS ${ARG_CPP_OPTIONS} + INCLUDE_DIR "${ARG_INCLUDE_DIR}" + THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}" + ) + elseif ("${lang}" STREQUAL "py" OR "${lang}" STREQUAL "python") + if (DEFINED ARG_PY_NAMESPACE) + set(namespace_args NAMESPACE "${ARG_PY_NAMESPACE}") + endif() + add_fbthrift_py_library( + "${LIB_NAME}_py" "${THRIFT_FILE}" + SERVICES ${ARG_SERVICES} + ${namespace_args} + DEPENDS ${PY_DEPENDS} + OPTIONS ${ARG_PY_OPTIONS} + THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}" + ) + else() + message( + FATAL_ERROR "unknown language for thrift library ${LIB_NAME}: ${lang}" + ) + endif() + endforeach() +endfunction() diff --git a/build/fbcode_builder/CMake/FBThriftPyLibrary.cmake b/build/fbcode_builder/CMake/FBThriftPyLibrary.cmake new file mode 100644 index 000000000000..7bd8879eedd7 --- /dev/null +++ b/build/fbcode_builder/CMake/FBThriftPyLibrary.cmake @@ -0,0 +1,111 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +include(FBCMakeParseArgs) +include(FBPythonBinary) + +# Generate a Python library from a thrift file +function(add_fbthrift_py_library LIB_NAME THRIFT_FILE) + # Parse the arguments + set(one_value_args NAMESPACE THRIFT_INCLUDE_DIR) + set(multi_value_args SERVICES DEPENDS OPTIONS) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + + if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR) + set(ARG_THRIFT_INCLUDE_DIR "include/thrift-files") + endif() + + get_filename_component(base ${THRIFT_FILE} NAME_WE) + set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE}-py") + + # Parse the namespace value + if (NOT DEFINED ARG_NAMESPACE) + set(ARG_NAMESPACE "${base}") + endif() + + string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}") + set(py_output_dir "${output_dir}/gen-py/${namespace_dir}") + list(APPEND generated_sources + "${py_output_dir}/__init__.py" + "${py_output_dir}/ttypes.py" + "${py_output_dir}/constants.py" + ) + foreach(service IN LISTS ARG_SERVICES) + list(APPEND generated_sources + ${py_output_dir}/${service}.py + ) + endforeach() + + # Define a dummy interface library to help propagate the thrift include + # directories between dependencies. + add_library("${LIB_NAME}.thrift_includes" INTERFACE) + target_include_directories( + "${LIB_NAME}.thrift_includes" + INTERFACE + "$" + "$" + ) + foreach(dep IN LISTS ARG_DEPENDS) + target_link_libraries( + "${LIB_NAME}.thrift_includes" + INTERFACE "${dep}.thrift_includes" + ) + endforeach() + + # This generator expression gets the list of include directories required + # for all of our dependencies. + # It requires using COMMAND_EXPAND_LISTS in the add_custom_command() call + # below. COMMAND_EXPAND_LISTS is only available in CMake 3.8+ + # If we really had to support older versions of CMake we would probably need + # to use a wrapper script around the thrift compiler that could take the + # include list as a single argument and split it up before invoking the + # thrift compiler. + if (NOT POLICY CMP0067) + message(FATAL_ERROR "add_fbthrift_py_library() requires CMake 3.8+") + endif() + set( + thrift_include_options + "-I;$,;-I;>" + ) + + # Always force generation of "new-style" python classes for Python 2 + list(APPEND ARG_OPTIONS "new_style") + # CMake 3.12 is finally getting a list(JOIN) function, but until then + # treating the list as a string and replacing the semicolons is good enough. + string(REPLACE ";" "," GEN_ARG_STR "${ARG_OPTIONS}") + + # Emit the rule to run the thrift compiler + add_custom_command( + OUTPUT + ${generated_sources} + COMMAND_EXPAND_LISTS + COMMAND + "${CMAKE_COMMAND}" -E make_directory "${output_dir}" + COMMAND + "${FBTHRIFT_COMPILER}" + --strict + --gen "py:${GEN_ARG_STR}" + "${thrift_include_options}" + -o "${output_dir}" + "${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}" + WORKING_DIRECTORY + "${CMAKE_BINARY_DIR}" + MAIN_DEPENDENCY + "${THRIFT_FILE}" + DEPENDS + "${FBTHRIFT_COMPILER}" + ) + + # We always want to pass the namespace as "" to this call: + # thrift will already emit the files with the desired namespace prefix under + # gen-py. We don't want add_fb_python_library() to prepend the namespace a + # second time. + add_fb_python_library( + "${LIB_NAME}" + BASE_DIR "${output_dir}/gen-py" + NAMESPACE "" + SOURCES ${generated_sources} + DEPENDS ${ARG_DEPENDS} FBThrift::thrift_py + ) +endfunction() diff --git a/build/fbcode_builder/CMake/FindGMock.cmake b/build/fbcode_builder/CMake/FindGMock.cmake new file mode 100644 index 000000000000..cd042dd9c4fa --- /dev/null +++ b/build/fbcode_builder/CMake/FindGMock.cmake @@ -0,0 +1,80 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Find libgmock +# +# LIBGMOCK_DEFINES - List of defines when using libgmock. +# LIBGMOCK_INCLUDE_DIR - where to find gmock/gmock.h, etc. +# LIBGMOCK_LIBRARIES - List of libraries when using libgmock. +# LIBGMOCK_FOUND - True if libgmock found. + +IF (LIBGMOCK_INCLUDE_DIR) + # Already in cache, be silent + SET(LIBGMOCK_FIND_QUIETLY TRUE) +ENDIF () + +find_package(GTest CONFIG QUIET) +if (TARGET GTest::gmock) + get_target_property(LIBGMOCK_DEFINES GTest::gtest INTERFACE_COMPILE_DEFINITIONS) + if (NOT ${LIBGMOCK_DEFINES}) + # Explicitly set to empty string if not found to avoid it being + # set to NOTFOUND and breaking compilation + set(LIBGMOCK_DEFINES "") + endif() + get_target_property(LIBGMOCK_INCLUDE_DIR GTest::gtest INTERFACE_INCLUDE_DIRECTORIES) + set(LIBGMOCK_LIBRARIES GTest::gmock_main GTest::gmock GTest::gtest) + set(LIBGMOCK_FOUND ON) + message(STATUS "Found gmock via config, defines=${LIBGMOCK_DEFINES}, include=${LIBGMOCK_INCLUDE_DIR}, libs=${LIBGMOCK_LIBRARIES}") +else() + + FIND_PATH(LIBGMOCK_INCLUDE_DIR gmock/gmock.h) + + FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_DEBUG NAMES gmock_maind) + FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_RELEASE NAMES gmock_main) + FIND_LIBRARY(LIBGMOCK_LIBRARY_DEBUG NAMES gmockd) + FIND_LIBRARY(LIBGMOCK_LIBRARY_RELEASE NAMES gmock) + FIND_LIBRARY(LIBGTEST_LIBRARY_DEBUG NAMES gtestd) + FIND_LIBRARY(LIBGTEST_LIBRARY_RELEASE NAMES gtest) + + find_package(Threads REQUIRED) + INCLUDE(SelectLibraryConfigurations) + SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK_MAIN) + SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK) + SELECT_LIBRARY_CONFIGURATIONS(LIBGTEST) + + set(LIBGMOCK_LIBRARIES + ${LIBGMOCK_MAIN_LIBRARY} + ${LIBGMOCK_LIBRARY} + ${LIBGTEST_LIBRARY} + Threads::Threads + ) + + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + # The GTEST_LINKED_AS_SHARED_LIBRARY macro must be set properly on Windows. + # + # There isn't currently an easy way to determine if a library was compiled as + # a shared library on Windows, so just assume we've been built against a + # shared build of gmock for now. + SET(LIBGMOCK_DEFINES "GTEST_LINKED_AS_SHARED_LIBRARY=1" CACHE STRING "") + endif() + + # handle the QUIETLY and REQUIRED arguments and set LIBGMOCK_FOUND to TRUE if + # all listed variables are TRUE + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS( + GMock + DEFAULT_MSG + LIBGMOCK_MAIN_LIBRARY + LIBGMOCK_LIBRARY + LIBGTEST_LIBRARY + LIBGMOCK_LIBRARIES + LIBGMOCK_INCLUDE_DIR + ) + + MARK_AS_ADVANCED( + LIBGMOCK_DEFINES + LIBGMOCK_MAIN_LIBRARY + LIBGMOCK_LIBRARY + LIBGTEST_LIBRARY + LIBGMOCK_LIBRARIES + LIBGMOCK_INCLUDE_DIR + ) +endif() diff --git a/build/fbcode_builder/CMake/FindGflags.cmake b/build/fbcode_builder/CMake/FindGflags.cmake new file mode 100644 index 000000000000..c00896a34398 --- /dev/null +++ b/build/fbcode_builder/CMake/FindGflags.cmake @@ -0,0 +1,105 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Find libgflags. +# There's a lot of compatibility cruft going on in here, both +# to deal with changes across the FB consumers of this and also +# to deal with variances in behavior of cmake itself. +# +# Since this file is named FindGflags.cmake the cmake convention +# is for the module to export both GFLAGS_FOUND and Gflags_FOUND. +# The convention expected by consumers is that we export the +# following variables, even though these do not match the cmake +# conventions: +# +# LIBGFLAGS_INCLUDE_DIR - where to find gflags/gflags.h, etc. +# LIBGFLAGS_LIBRARY - List of libraries when using libgflags. +# LIBGFLAGS_FOUND - True if libgflags found. +# +# We need to be able to locate gflags both from an installed +# cmake config file and just from the raw headers and libs, so +# test for the former and then the latter, and then stick +# the results together and export them into the variables +# listed above. +# +# For forwards compatibility, we export the following variables: +# +# gflags_INCLUDE_DIR - where to find gflags/gflags.h, etc. +# gflags_TARGET / GFLAGS_TARGET / gflags_LIBRARIES +# - List of libraries when using libgflags. +# gflags_FOUND - True if libgflags found. +# + +IF (LIBGFLAGS_INCLUDE_DIR) + # Already in cache, be silent + SET(Gflags_FIND_QUIETLY TRUE) +ENDIF () + +find_package(gflags CONFIG QUIET) +if (gflags_FOUND) + if (NOT Gflags_FIND_QUIETLY) + message(STATUS "Found gflags from package config ${gflags_CONFIG}") + endif() + # Re-export the config-specified libs with our local names + set(LIBGFLAGS_LIBRARY ${gflags_LIBRARIES}) + set(LIBGFLAGS_INCLUDE_DIR ${gflags_INCLUDE_DIR}) + if(NOT EXISTS "${gflags_INCLUDE_DIR}") + # The gflags-devel RPM on recent RedHat-based systems is somewhat broken. + # RedHat symlinks /lib64 to /usr/lib64, and this breaks some of the + # relative path computation performed in gflags-config.cmake. The package + # config file ends up being found via /lib64, but the relative path + # computation it does only works if it was found in /usr/lib64. + # If gflags_INCLUDE_DIR does not actually exist, simply default it to + # /usr/include on these systems. + set(LIBGFLAGS_INCLUDE_DIR "/usr/include") + endif() + set(LIBGFLAGS_FOUND ${gflags_FOUND}) + # cmake module compat + set(GFLAGS_FOUND ${gflags_FOUND}) + set(Gflags_FOUND ${gflags_FOUND}) +else() + FIND_PATH(LIBGFLAGS_INCLUDE_DIR gflags/gflags.h) + + FIND_LIBRARY(LIBGFLAGS_LIBRARY_DEBUG NAMES gflagsd gflags_staticd) + FIND_LIBRARY(LIBGFLAGS_LIBRARY_RELEASE NAMES gflags gflags_static) + + INCLUDE(SelectLibraryConfigurations) + SELECT_LIBRARY_CONFIGURATIONS(LIBGFLAGS) + + # handle the QUIETLY and REQUIRED arguments and set LIBGFLAGS_FOUND to TRUE if + # all listed variables are TRUE + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(gflags DEFAULT_MSG LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR) + # cmake module compat + set(Gflags_FOUND ${GFLAGS_FOUND}) + # compat with some existing FindGflags consumers + set(LIBGFLAGS_FOUND ${GFLAGS_FOUND}) + + # Compat with the gflags CONFIG based detection + set(gflags_FOUND ${GFLAGS_FOUND}) + set(gflags_INCLUDE_DIR ${LIBGFLAGS_INCLUDE_DIR}) + set(gflags_LIBRARIES ${LIBGFLAGS_LIBRARY}) + set(GFLAGS_TARGET ${LIBGFLAGS_LIBRARY}) + set(gflags_TARGET ${LIBGFLAGS_LIBRARY}) + + MARK_AS_ADVANCED(LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR) +endif() + +# Compat with the gflags CONFIG based detection +if (LIBGFLAGS_FOUND AND NOT TARGET gflags) + add_library(gflags UNKNOWN IMPORTED) + if(TARGET gflags-shared) + # If the installed gflags CMake package config defines a gflags-shared + # target but not gflags, just make the gflags target that we define + # depend on the gflags-shared target. + target_link_libraries(gflags INTERFACE gflags-shared) + # Export LIBGFLAGS_LIBRARY as the gflags-shared target in this case. + set(LIBGFLAGS_LIBRARY gflags-shared) + else() + set_target_properties( + gflags + PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${LIBGFLAGS_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${LIBGFLAGS_INCLUDE_DIR}" + ) + endif() +endif() diff --git a/build/fbcode_builder/CMake/FindGlog.cmake b/build/fbcode_builder/CMake/FindGlog.cmake new file mode 100644 index 000000000000..752647cb3357 --- /dev/null +++ b/build/fbcode_builder/CMake/FindGlog.cmake @@ -0,0 +1,37 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# - Try to find Glog +# Once done, this will define +# +# GLOG_FOUND - system has Glog +# GLOG_INCLUDE_DIRS - the Glog include directories +# GLOG_LIBRARIES - link these to use Glog + +include(FindPackageHandleStandardArgs) +include(SelectLibraryConfigurations) + +find_library(GLOG_LIBRARY_RELEASE glog + PATHS ${GLOG_LIBRARYDIR}) +find_library(GLOG_LIBRARY_DEBUG glogd + PATHS ${GLOG_LIBRARYDIR}) + +find_path(GLOG_INCLUDE_DIR glog/logging.h + PATHS ${GLOG_INCLUDEDIR}) + +select_library_configurations(GLOG) + +find_package_handle_standard_args(glog DEFAULT_MSG + GLOG_LIBRARY + GLOG_INCLUDE_DIR) + +mark_as_advanced( + GLOG_LIBRARY + GLOG_INCLUDE_DIR) + +set(GLOG_LIBRARIES ${GLOG_LIBRARY}) +set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) + +if (NOT TARGET glog::glog) + add_library(glog::glog UNKNOWN IMPORTED) + set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIRS}") + set_target_properties(glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${GLOG_LIBRARIES}") +endif() diff --git a/build/fbcode_builder/CMake/FindLibEvent.cmake b/build/fbcode_builder/CMake/FindLibEvent.cmake new file mode 100644 index 000000000000..dd11ebd8435d --- /dev/null +++ b/build/fbcode_builder/CMake/FindLibEvent.cmake @@ -0,0 +1,77 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# - Find LibEvent (a cross event library) +# This module defines +# LIBEVENT_INCLUDE_DIR, where to find LibEvent headers +# LIBEVENT_LIB, LibEvent libraries +# LibEvent_FOUND, If false, do not try to use libevent + +set(LibEvent_EXTRA_PREFIXES /usr/local /opt/local "$ENV{HOME}") +foreach(prefix ${LibEvent_EXTRA_PREFIXES}) + list(APPEND LibEvent_INCLUDE_PATHS "${prefix}/include") + list(APPEND LibEvent_LIB_PATHS "${prefix}/lib") +endforeach() + +find_package(Libevent CONFIG QUIET) +if (TARGET event) + # Re-export the config under our own names + + # Somewhat gross, but some vcpkg installed libevents have a relative + # `include` path exported into LIBEVENT_INCLUDE_DIRS, which triggers + # a cmake error because it resolves to the `include` dir within the + # folly repo, which is not something cmake allows to be in the + # INTERFACE_INCLUDE_DIRECTORIES. Thankfully on such a system the + # actual include directory is already part of the global include + # directories, so we can just skip it. + if (NOT "${LIBEVENT_INCLUDE_DIRS}" STREQUAL "include") + set(LIBEVENT_INCLUDE_DIR ${LIBEVENT_INCLUDE_DIRS}) + else() + set(LIBEVENT_INCLUDE_DIR) + endif() + + # Unfortunately, with a bare target name `event`, downstream consumers + # of the package that depends on `Libevent` located via CONFIG end + # up exporting just a bare `event` in their libraries. This is problematic + # because this in interpreted as just `-levent` with no library path. + # When libevent is not installed in the default installation prefix + # this results in linker errors. + # To resolve this, we ask cmake to lookup the full path to the library + # and use that instead. + cmake_policy(PUSH) + if(POLICY CMP0026) + # Allow reading the LOCATION property + cmake_policy(SET CMP0026 OLD) + endif() + get_target_property(LIBEVENT_LIB event LOCATION) + cmake_policy(POP) + + set(LibEvent_FOUND ${Libevent_FOUND}) + if (NOT LibEvent_FIND_QUIETLY) + message(STATUS "Found libevent from package config include=${LIBEVENT_INCLUDE_DIRS} lib=${LIBEVENT_LIB}") + endif() +else() + find_path(LIBEVENT_INCLUDE_DIR event.h PATHS ${LibEvent_INCLUDE_PATHS}) + find_library(LIBEVENT_LIB NAMES event PATHS ${LibEvent_LIB_PATHS}) + + if (LIBEVENT_LIB AND LIBEVENT_INCLUDE_DIR) + set(LibEvent_FOUND TRUE) + set(LIBEVENT_LIB ${LIBEVENT_LIB}) + else () + set(LibEvent_FOUND FALSE) + endif () + + if (LibEvent_FOUND) + if (NOT LibEvent_FIND_QUIETLY) + message(STATUS "Found libevent: ${LIBEVENT_LIB}") + endif () + else () + if (LibEvent_FIND_REQUIRED) + message(FATAL_ERROR "Could NOT find libevent.") + endif () + message(STATUS "libevent NOT found.") + endif () + + mark_as_advanced( + LIBEVENT_LIB + LIBEVENT_INCLUDE_DIR + ) +endif() diff --git a/build/fbcode_builder/CMake/FindLibUnwind.cmake b/build/fbcode_builder/CMake/FindLibUnwind.cmake new file mode 100644 index 000000000000..b01a674a5ba0 --- /dev/null +++ b/build/fbcode_builder/CMake/FindLibUnwind.cmake @@ -0,0 +1,29 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +find_path(LIBUNWIND_INCLUDE_DIR NAMES libunwind.h) +mark_as_advanced(LIBUNWIND_INCLUDE_DIR) + +find_library(LIBUNWIND_LIBRARY NAMES unwind) +mark_as_advanced(LIBUNWIND_LIBRARY) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS( + LIBUNWIND + REQUIRED_VARS LIBUNWIND_LIBRARY LIBUNWIND_INCLUDE_DIR) + +if(LIBUNWIND_FOUND) + set(LIBUNWIND_LIBRARIES ${LIBUNWIND_LIBRARY}) + set(LIBUNWIND_INCLUDE_DIRS ${LIBUNWIND_INCLUDE_DIR}) +endif() diff --git a/build/fbcode_builder/CMake/FindPCRE.cmake b/build/fbcode_builder/CMake/FindPCRE.cmake new file mode 100644 index 000000000000..32ccb372536f --- /dev/null +++ b/build/fbcode_builder/CMake/FindPCRE.cmake @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +include(FindPackageHandleStandardArgs) +find_path(PCRE_INCLUDE_DIR NAMES pcre.h) +find_library(PCRE_LIBRARY NAMES pcre) +find_package_handle_standard_args( + PCRE + DEFAULT_MSG + PCRE_LIBRARY + PCRE_INCLUDE_DIR +) +mark_as_advanced(PCRE_INCLUDE_DIR PCRE_LIBRARY) diff --git a/build/fbcode_builder/CMake/FindRe2.cmake b/build/fbcode_builder/CMake/FindRe2.cmake new file mode 100644 index 000000000000..013ae7761e9c --- /dev/null +++ b/build/fbcode_builder/CMake/FindRe2.cmake @@ -0,0 +1,20 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2. + +find_library(RE2_LIBRARY re2) +mark_as_advanced(RE2_LIBRARY) + +find_path(RE2_INCLUDE_DIR NAMES re2/re2.h) +mark_as_advanced(RE2_INCLUDE_DIR) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS( + RE2 + REQUIRED_VARS RE2_LIBRARY RE2_INCLUDE_DIR) + +if(RE2_FOUND) + set(RE2_LIBRARY ${RE2_LIBRARY}) + set(RE2_INCLUDE_DIR, ${RE2_INCLUDE_DIR}) +endif() diff --git a/build/fbcode_builder/CMake/FindSodium.cmake b/build/fbcode_builder/CMake/FindSodium.cmake new file mode 100644 index 000000000000..3c3f1245c1dc --- /dev/null +++ b/build/fbcode_builder/CMake/FindSodium.cmake @@ -0,0 +1,297 @@ +# Written in 2016 by Henrik Steffen Gaßmann +# +# To the extent possible under law, the author(s) have dedicated all +# copyright and related and neighboring rights to this software to the +# public domain worldwide. This software is distributed without any warranty. +# +# You should have received a copy of the CC0 Public Domain Dedication +# along with this software. If not, see +# +# http://creativecommons.org/publicdomain/zero/1.0/ +# +######################################################################## +# Tries to find the local libsodium installation. +# +# On Windows the sodium_DIR environment variable is used as a default +# hint which can be overridden by setting the corresponding cmake variable. +# +# Once done the following variables will be defined: +# +# sodium_FOUND +# sodium_INCLUDE_DIR +# sodium_LIBRARY_DEBUG +# sodium_LIBRARY_RELEASE +# +# +# Furthermore an imported "sodium" target is created. +# + +if (CMAKE_C_COMPILER_ID STREQUAL "GNU" + OR CMAKE_C_COMPILER_ID STREQUAL "Clang") + set(_GCC_COMPATIBLE 1) +endif() + +# static library option +if (NOT DEFINED sodium_USE_STATIC_LIBS) + option(sodium_USE_STATIC_LIBS "enable to statically link against sodium" OFF) +endif() +if(NOT (sodium_USE_STATIC_LIBS EQUAL sodium_USE_STATIC_LIBS_LAST)) + unset(sodium_LIBRARY CACHE) + unset(sodium_LIBRARY_DEBUG CACHE) + unset(sodium_LIBRARY_RELEASE CACHE) + unset(sodium_DLL_DEBUG CACHE) + unset(sodium_DLL_RELEASE CACHE) + set(sodium_USE_STATIC_LIBS_LAST ${sodium_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable") +endif() + + +######################################################################## +# UNIX +if (UNIX) + # import pkg-config + find_package(PkgConfig QUIET) + if (PKG_CONFIG_FOUND) + pkg_check_modules(sodium_PKG QUIET libsodium) + endif() + + if(sodium_USE_STATIC_LIBS) + foreach(_libname ${sodium_PKG_STATIC_LIBRARIES}) + if (NOT _libname MATCHES "^lib.*\\.a$") # ignore strings already ending with .a + list(INSERT sodium_PKG_STATIC_LIBRARIES 0 "lib${_libname}.a") + endif() + endforeach() + list(REMOVE_DUPLICATES sodium_PKG_STATIC_LIBRARIES) + + # if pkgconfig for libsodium doesn't provide + # static lib info, then override PKG_STATIC here.. + if (NOT sodium_PKG_STATIC_FOUND) + set(sodium_PKG_STATIC_LIBRARIES libsodium.a) + endif() + + set(XPREFIX sodium_PKG_STATIC) + else() + if (NOT sodium_PKG_FOUND) + set(sodium_PKG_LIBRARIES sodium) + endif() + + set(XPREFIX sodium_PKG) + endif() + + find_path(sodium_INCLUDE_DIR sodium.h + HINTS ${${XPREFIX}_INCLUDE_DIRS} + ) + find_library(sodium_LIBRARY_DEBUG NAMES ${${XPREFIX}_LIBRARIES} + HINTS ${${XPREFIX}_LIBRARY_DIRS} + ) + find_library(sodium_LIBRARY_RELEASE NAMES ${${XPREFIX}_LIBRARIES} + HINTS ${${XPREFIX}_LIBRARY_DIRS} + ) + + +######################################################################## +# Windows +elseif (WIN32) + set(sodium_DIR "$ENV{sodium_DIR}" CACHE FILEPATH "sodium install directory") + mark_as_advanced(sodium_DIR) + + find_path(sodium_INCLUDE_DIR sodium.h + HINTS ${sodium_DIR} + PATH_SUFFIXES include + ) + + if (MSVC) + # detect target architecture + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" [=[ + #if defined _M_IX86 + #error ARCH_VALUE x86_32 + #elif defined _M_X64 + #error ARCH_VALUE x86_64 + #endif + #error ARCH_VALUE unknown + ]=]) + try_compile(_UNUSED_VAR "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" + OUTPUT_VARIABLE _COMPILATION_LOG + ) + string(REGEX REPLACE ".*ARCH_VALUE ([a-zA-Z0-9_]+).*" "\\1" _TARGET_ARCH "${_COMPILATION_LOG}") + + # construct library path + if (_TARGET_ARCH STREQUAL "x86_32") + string(APPEND _PLATFORM_PATH "Win32") + elseif(_TARGET_ARCH STREQUAL "x86_64") + string(APPEND _PLATFORM_PATH "x64") + else() + message(FATAL_ERROR "the ${_TARGET_ARCH} architecture is not supported by Findsodium.cmake.") + endif() + string(APPEND _PLATFORM_PATH "/$$CONFIG$$") + + if (MSVC_VERSION LESS 1900) + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60") + else() + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50") + endif() + string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}") + + if (sodium_USE_STATIC_LIBS) + string(APPEND _PLATFORM_PATH "/static") + else() + string(APPEND _PLATFORM_PATH "/dynamic") + endif() + + string(REPLACE "$$CONFIG$$" "Debug" _DEBUG_PATH_SUFFIX "${_PLATFORM_PATH}") + string(REPLACE "$$CONFIG$$" "Release" _RELEASE_PATH_SUFFIX "${_PLATFORM_PATH}") + + find_library(sodium_LIBRARY_DEBUG libsodium.lib + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} + ) + find_library(sodium_LIBRARY_RELEASE libsodium.lib + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} + ) + if (NOT sodium_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll") + find_library(sodium_DLL_DEBUG libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} + ) + find_library(sodium_DLL_RELEASE libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} + ) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK}) + endif() + + elseif(_GCC_COMPATIBLE) + if (sodium_USE_STATIC_LIBS) + find_library(sodium_LIBRARY_DEBUG libsodium.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + find_library(sodium_LIBRARY_RELEASE libsodium.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + else() + find_library(sodium_LIBRARY_DEBUG libsodium.dll.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + find_library(sodium_LIBRARY_RELEASE libsodium.dll.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + + file(GLOB _DLL + LIST_DIRECTORIES false + RELATIVE "${sodium_DIR}/bin" + "${sodium_DIR}/bin/libsodium*.dll" + ) + find_library(sodium_DLL_DEBUG ${_DLL} libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES bin + ) + find_library(sodium_DLL_RELEASE ${_DLL} libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES bin + ) + endif() + else() + message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") + endif() + + +######################################################################## +# unsupported +else() + message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") +endif() + + +######################################################################## +# common stuff + +# extract sodium version +if (sodium_INCLUDE_DIR) + set(_VERSION_HEADER "${_INCLUDE_DIR}/sodium/version.h") + if (EXISTS _VERSION_HEADER) + file(READ "${_VERSION_HEADER}" _VERSION_HEADER_CONTENT) + string(REGEX REPLACE ".*#[ \t]*define[ \t]*SODIUM_VERSION_STRING[ \t]*\"([^\n]*)\".*" "\\1" + sodium_VERSION "${_VERSION_HEADER_CONTENT}") + set(sodium_VERSION "${sodium_VERSION}" PARENT_SCOPE) + endif() +endif() + +# communicate results +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + Sodium # The name must be either uppercase or match the filename case. + REQUIRED_VARS + sodium_LIBRARY_RELEASE + sodium_LIBRARY_DEBUG + sodium_INCLUDE_DIR + VERSION_VAR + sodium_VERSION +) + +if(Sodium_FOUND) + set(sodium_LIBRARIES + optimized ${sodium_LIBRARY_RELEASE} debug ${sodium_LIBRARY_DEBUG}) +endif() + +# mark file paths as advanced +mark_as_advanced(sodium_INCLUDE_DIR) +mark_as_advanced(sodium_LIBRARY_DEBUG) +mark_as_advanced(sodium_LIBRARY_RELEASE) +if (WIN32) + mark_as_advanced(sodium_DLL_DEBUG) + mark_as_advanced(sodium_DLL_RELEASE) +endif() + +# create imported target +if(sodium_USE_STATIC_LIBS) + set(_LIB_TYPE STATIC) +else() + set(_LIB_TYPE SHARED) +endif() + +if(NOT TARGET sodium) + add_library(sodium ${_LIB_TYPE} IMPORTED) +endif() + +set_target_properties(sodium PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" +) + +if (sodium_USE_STATIC_LIBS) + set_target_properties(sodium PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC" + IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}" + ) +else() + if (UNIX) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}" + ) + elseif (WIN32) + set_target_properties(sodium PROPERTIES + IMPORTED_IMPLIB "${sodium_LIBRARY_RELEASE}" + IMPORTED_IMPLIB_DEBUG "${sodium_LIBRARY_DEBUG}" + ) + if (NOT (sodium_DLL_DEBUG MATCHES ".*-NOTFOUND")) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION_DEBUG "${sodium_DLL_DEBUG}" + ) + endif() + if (NOT (sodium_DLL_RELEASE MATCHES ".*-NOTFOUND")) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}" + ) + endif() + endif() +endif() diff --git a/build/fbcode_builder/CMake/FindZstd.cmake b/build/fbcode_builder/CMake/FindZstd.cmake new file mode 100644 index 000000000000..89300ddfd398 --- /dev/null +++ b/build/fbcode_builder/CMake/FindZstd.cmake @@ -0,0 +1,41 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# - Try to find Facebook zstd library +# This will define +# ZSTD_FOUND +# ZSTD_INCLUDE_DIR +# ZSTD_LIBRARY +# + +find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + +find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd) +find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static) + +include(SelectLibraryConfigurations) +SELECT_LIBRARY_CONFIGURATIONS(ZSTD) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS( + ZSTD DEFAULT_MSG + ZSTD_LIBRARY ZSTD_INCLUDE_DIR +) + +if (ZSTD_FOUND) + message(STATUS "Found Zstd: ${ZSTD_LIBRARY}") +endif() + +mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) diff --git a/build/fbcode_builder/CMake/RustStaticLibrary.cmake b/build/fbcode_builder/CMake/RustStaticLibrary.cmake new file mode 100644 index 000000000000..8546fe2fbb12 --- /dev/null +++ b/build/fbcode_builder/CMake/RustStaticLibrary.cmake @@ -0,0 +1,291 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +include(FBCMakeParseArgs) + +set( + USE_CARGO_VENDOR AUTO CACHE STRING + "Download Rust Crates from an internally vendored location" +) +set_property(CACHE USE_CARGO_VENDOR PROPERTY STRINGS AUTO ON OFF) + +set(RUST_VENDORED_CRATES_DIR "$ENV{RUST_VENDORED_CRATES_DIR}") +if("${USE_CARGO_VENDOR}" STREQUAL "AUTO") + if(EXISTS "${RUST_VENDORED_CRATES_DIR}") + set(USE_CARGO_VENDOR ON) + else() + set(USE_CARGO_VENDOR OFF) + endif() +endif() + +if(USE_CARGO_VENDOR) + if(NOT EXISTS "${RUST_VENDORED_CRATES_DIR}") + message( + FATAL "vendored rust crates not present: " + "${RUST_VENDORED_CRATES_DIR}" + ) + endif() + + set(RUST_CARGO_HOME "${CMAKE_BINARY_DIR}/_cargo_home") + file(MAKE_DIRECTORY "${RUST_CARGO_HOME}") + + file( + TO_NATIVE_PATH "${RUST_VENDORED_CRATES_DIR}" + ESCAPED_RUST_VENDORED_CRATES_DIR + ) + string( + REPLACE "\\" "\\\\" + ESCAPED_RUST_VENDORED_CRATES_DIR + "${ESCAPED_RUST_VENDORED_CRATES_DIR}" + ) + file( + WRITE "${RUST_CARGO_HOME}/config" + "[source.crates-io]\n" + "replace-with = \"vendored-sources\"\n" + "\n" + "[source.vendored-sources]\n" + "directory = \"${ESCAPED_RUST_VENDORED_CRATES_DIR}\"\n" + ) +endif() + +# Cargo is a build system in itself, and thus will try to take advantage of all +# the cores on the system. Unfortunately, this conflicts with Ninja, since it +# also tries to utilize all the cores. This can lead to a system that is +# completely overloaded with compile jobs to the point where nothing else can +# be achieved on the system. +# +# Let's inform Ninja of this fact so it won't try to spawn other jobs while +# Rust being compiled. +set_property(GLOBAL APPEND PROPERTY JOB_POOLS rust_job_pool=1) + +# This function creates an interface library target based on the static library +# built by Cargo. It will call Cargo to build a staticlib and generate a CMake +# interface library with it. +# +# This function requires `find_package(Python COMPONENTS Interpreter)`. +# +# You need to set `lib:crate-type = ["staticlib"]` in your Cargo.toml to make +# Cargo build static library. +# +# ```cmake +# rust_static_library( [CRATE ]) +# ``` +# +# Parameters: +# - TARGET: +# Name of the target name. This function will create an interface library +# target with this name. +# - CRATE_NAME: +# Name of the crate. This parameter is optional. If unspecified, it will +# fallback to `${TARGET}`. +# +# This function creates two targets: +# - "${TARGET}": an interface library target contains the static library built +# from Cargo. +# - "${TARGET}.cargo": an internal custom target that invokes Cargo. +# +# If you are going to use this static library from C/C++, you will need to +# write header files for the library (or generate with cbindgen) and bind these +# headers with the interface library. +# +function(rust_static_library TARGET) + fb_cmake_parse_args(ARG "" "CRATE" "" "${ARGN}") + + if(DEFINED ARG_CRATE) + set(crate_name "${ARG_CRATE}") + else() + set(crate_name "${TARGET}") + endif() + + set(cargo_target "${TARGET}.cargo") + set(target_dir $,debug,release>) + set(staticlib_name "${CMAKE_STATIC_LIBRARY_PREFIX}${crate_name}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(rust_staticlib "${CMAKE_CURRENT_BINARY_DIR}/${target_dir}/${staticlib_name}") + + set(cargo_cmd cargo) + if(WIN32) + set(cargo_cmd cargo.exe) + endif() + + set(cargo_flags build $,,--release> -p ${crate_name}) + if(USE_CARGO_VENDOR) + set(extra_cargo_env "CARGO_HOME=${RUST_CARGO_HOME}") + set(cargo_flags ${cargo_flags}) + endif() + + add_custom_target( + ${cargo_target} + COMMAND + "${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock" + COMMAND + "${CMAKE_COMMAND}" -E env + "CARGO_TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR}" + ${extra_cargo_env} + ${cargo_cmd} + ${cargo_flags} + COMMENT "Building Rust crate '${crate_name}'..." + JOB_POOL rust_job_pool + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + BYPRODUCTS + "${CMAKE_CURRENT_BINARY_DIR}/debug/${staticlib_name}" + "${CMAKE_CURRENT_BINARY_DIR}/release/${staticlib_name}" + ) + + add_library(${TARGET} INTERFACE) + add_dependencies(${TARGET} ${cargo_target}) + set_target_properties( + ${TARGET} + PROPERTIES + INTERFACE_STATICLIB_OUTPUT_PATH "${rust_staticlib}" + INTERFACE_INSTALL_LIBNAME + "${CMAKE_STATIC_LIBRARY_PREFIX}${crate_name}_rs${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + target_link_libraries( + ${TARGET} + INTERFACE "$" + ) +endfunction() + +# This function instructs cmake to define a target that will use `cargo build` +# to build a bin crate referenced by the Cargo.toml file in the current source +# directory. +# It accepts a single `TARGET` parameter which will be passed as the package +# name to `cargo build -p TARGET`. If binary has different name as package, +# use optional flag BINARY_NAME to override it. +# The cmake target will be registered to build by default as part of the +# ALL target. +function(rust_executable TARGET) + fb_cmake_parse_args(ARG "" "BINARY_NAME" "" "${ARGN}") + + set(crate_name "${TARGET}") + set(cargo_target "${TARGET}.cargo") + set(target_dir $,debug,release>) + + if(DEFINED ARG_BINARY_NAME) + set(executable_name "${ARG_BINARY_NAME}${CMAKE_EXECUTABLE_SUFFIX}") + else() + set(executable_name "${crate_name}${CMAKE_EXECUTABLE_SUFFIX}") + endif() + + set(cargo_cmd cargo) + if(WIN32) + set(cargo_cmd cargo.exe) + endif() + + set(cargo_flags build $,,--release> -p ${crate_name}) + if(USE_CARGO_VENDOR) + set(extra_cargo_env "CARGO_HOME=${RUST_CARGO_HOME}") + set(cargo_flags ${cargo_flags}) + endif() + + add_custom_target( + ${cargo_target} + ALL + COMMAND + "${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock" + COMMAND + "${CMAKE_COMMAND}" -E env + "CARGO_TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR}" + ${extra_cargo_env} + ${cargo_cmd} + ${cargo_flags} + COMMENT "Building Rust executable '${crate_name}'..." + JOB_POOL rust_job_pool + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + BYPRODUCTS + "${CMAKE_CURRENT_BINARY_DIR}/debug/${executable_name}" + "${CMAKE_CURRENT_BINARY_DIR}/release/${executable_name}" + ) + + set_property(TARGET "${cargo_target}" + PROPERTY EXECUTABLE "${CMAKE_CURRENT_BINARY_DIR}/${target_dir}/${executable_name}") +endfunction() + +# This function can be used to install the executable generated by a prior +# call to the `rust_executable` function. +# It requires a `TARGET` parameter to identify the target to be installed, +# and an optional `DESTINATION` parameter to specify the installation +# directory. If DESTINATION is not specified then the `bin` directory +# will be assumed. +function(install_rust_executable TARGET) + # Parse the arguments + set(one_value_args DESTINATION) + set(multi_value_args) + fb_cmake_parse_args( + ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" + ) + + if(NOT DEFINED ARG_DESTINATION) + set(ARG_DESTINATION bin) + endif() + + get_target_property(foo "${TARGET}.cargo" EXECUTABLE) + + install( + PROGRAMS "${foo}" + DESTINATION "${ARG_DESTINATION}" + ) +endfunction() + +# This function installs the interface target generated from the function +# `rust_static_library`. Use this function if you want to export your Rust +# target to external CMake targets. +# +# ```cmake +# install_rust_static_library( +# +# INSTALL_DIR +# [EXPORT ] +# ) +# ``` +# +# Parameters: +# - TARGET: Name of the Rust static library target. +# - EXPORT_NAME: Name of the exported target. +# - INSTALL_DIR: Path to the directory where this library will be installed. +# +function(install_rust_static_library TARGET) + fb_cmake_parse_args(ARG "" "EXPORT;INSTALL_DIR" "" "${ARGN}") + + get_property( + staticlib_output_path + TARGET "${TARGET}" + PROPERTY INTERFACE_STATICLIB_OUTPUT_PATH + ) + get_property( + staticlib_output_name + TARGET "${TARGET}" + PROPERTY INTERFACE_INSTALL_LIBNAME + ) + + if(NOT DEFINED staticlib_output_path) + message(FATAL_ERROR "Not a rust_static_library target.") + endif() + + if(NOT DEFINED ARG_INSTALL_DIR) + message(FATAL_ERROR "Missing required argument.") + endif() + + if(DEFINED ARG_EXPORT) + set(install_export_args EXPORT "${ARG_EXPORT}") + endif() + + set(install_interface_dir "${ARG_INSTALL_DIR}") + if(NOT IS_ABSOLUTE "${install_interface_dir}") + set(install_interface_dir "\${_IMPORT_PREFIX}/${install_interface_dir}") + endif() + + target_link_libraries( + ${TARGET} INTERFACE + "$" + ) + install( + TARGETS ${TARGET} + ${install_export_args} + LIBRARY DESTINATION ${ARG_INSTALL_DIR} + ) + install( + FILES ${staticlib_output_path} + RENAME ${staticlib_output_name} + DESTINATION ${ARG_INSTALL_DIR} + ) +endfunction() diff --git a/build/fbcode_builder/CMake/fb_py_test_main.py b/build/fbcode_builder/CMake/fb_py_test_main.py new file mode 100644 index 000000000000..1f3563affe74 --- /dev/null +++ b/build/fbcode_builder/CMake/fb_py_test_main.py @@ -0,0 +1,820 @@ +#!/usr/bin/env python +# +# Copyright (c) Facebook, Inc. and its affiliates. +# +""" +This file contains the main module code for Python test programs. +""" + +from __future__ import print_function + +import contextlib +import ctypes +import fnmatch +import json +import logging +import optparse +import os +import platform +import re +import sys +import tempfile +import time +import traceback +import unittest +import warnings + +# Hide warning about importing "imp"; remove once python2 is gone. +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + import imp + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO +try: + import coverage +except ImportError: + coverage = None # type: ignore +try: + from importlib.machinery import SourceFileLoader +except ImportError: + SourceFileLoader = None # type: ignore + + +class get_cpu_instr_counter(object): + def read(self): + # TODO + return 0 + + +EXIT_CODE_SUCCESS = 0 +EXIT_CODE_TEST_FAILURE = 70 + + +class TestStatus(object): + + ABORTED = "FAILURE" + PASSED = "SUCCESS" + FAILED = "FAILURE" + EXPECTED_FAILURE = "SUCCESS" + UNEXPECTED_SUCCESS = "FAILURE" + SKIPPED = "ASSUMPTION_VIOLATION" + + +class PathMatcher(object): + def __init__(self, include_patterns, omit_patterns): + self.include_patterns = include_patterns + self.omit_patterns = omit_patterns + + def omit(self, path): + """ + Omit iff matches any of the omit_patterns or the include patterns are + not empty and none is matched + """ + path = os.path.realpath(path) + return any(fnmatch.fnmatch(path, p) for p in self.omit_patterns) or ( + self.include_patterns + and not any(fnmatch.fnmatch(path, p) for p in self.include_patterns) + ) + + def include(self, path): + return not self.omit(path) + + +class DebugWipeFinder(object): + """ + PEP 302 finder that uses a DebugWipeLoader for all files which do not need + coverage + """ + + def __init__(self, matcher): + self.matcher = matcher + + def find_module(self, fullname, path=None): + _, _, basename = fullname.rpartition(".") + try: + fd, pypath, (_, _, kind) = imp.find_module(basename, path) + except Exception: + # Finding without hooks using the imp module failed. One reason + # could be that there is a zip file on sys.path. The imp module + # does not support loading from there. Leave finding this module to + # the others finders in sys.meta_path. + return None + + if hasattr(fd, "close"): + fd.close() + if kind != imp.PY_SOURCE: + return None + if self.matcher.include(pypath): + return None + + """ + This is defined to match CPython's PyVarObject struct + """ + + class PyVarObject(ctypes.Structure): + _fields_ = [ + ("ob_refcnt", ctypes.c_long), + ("ob_type", ctypes.c_void_p), + ("ob_size", ctypes.c_ulong), + ] + + class DebugWipeLoader(SourceFileLoader): + """ + PEP302 loader that zeros out debug information before execution + """ + + def get_code(self, fullname): + code = super(DebugWipeLoader, self).get_code(fullname) + if code: + # Ideally we'd do + # code.co_lnotab = b'' + # But code objects are READONLY. Not to worry though; we'll + # directly modify CPython's object + code_impl = PyVarObject.from_address(id(code.co_lnotab)) + code_impl.ob_size = 0 + return code + + return DebugWipeLoader(fullname, pypath) + + +def optimize_for_coverage(cov, include_patterns, omit_patterns): + """ + We get better performance if we zero out debug information for files which + we're not interested in. Only available in CPython 3.3+ + """ + matcher = PathMatcher(include_patterns, omit_patterns) + if SourceFileLoader and platform.python_implementation() == "CPython": + sys.meta_path.insert(0, DebugWipeFinder(matcher)) + + +class TeeStream(object): + def __init__(self, *streams): + self._streams = streams + + def write(self, data): + for stream in self._streams: + stream.write(data) + + def flush(self): + for stream in self._streams: + stream.flush() + + def isatty(self): + return False + + +class CallbackStream(object): + def __init__(self, callback, bytes_callback=None, orig=None): + self._callback = callback + self._fileno = orig.fileno() if orig else None + + # Python 3 APIs: + # - `encoding` is a string holding the encoding name + # - `errors` is a string holding the error-handling mode for encoding + # - `buffer` should look like an io.BufferedIOBase object + + self.errors = orig.errors if orig else None + if bytes_callback: + # those members are only on the io.TextIOWrapper + self.encoding = orig.encoding if orig else "UTF-8" + self.buffer = CallbackStream(bytes_callback, orig=orig) + + def write(self, data): + self._callback(data) + + def flush(self): + pass + + def isatty(self): + return False + + def fileno(self): + return self._fileno + + +class BuckTestResult(unittest._TextTestResult): + """ + Our own TestResult class that outputs data in a format that can be easily + parsed by buck's test runner. + """ + + _instr_counter = get_cpu_instr_counter() + + def __init__( + self, stream, descriptions, verbosity, show_output, main_program, suite + ): + super(BuckTestResult, self).__init__(stream, descriptions, verbosity) + self._main_program = main_program + self._suite = suite + self._results = [] + self._current_test = None + self._saved_stdout = sys.stdout + self._saved_stderr = sys.stderr + self._show_output = show_output + + def getResults(self): + return self._results + + def startTest(self, test): + super(BuckTestResult, self).startTest(test) + + # Pass in the real stdout and stderr filenos. We can't really do much + # here to intercept callers who directly operate on these fileno + # objects. + sys.stdout = CallbackStream( + self.addStdout, self.addStdoutBytes, orig=sys.stdout + ) + sys.stderr = CallbackStream( + self.addStderr, self.addStderrBytes, orig=sys.stderr + ) + self._current_test = test + self._test_start_time = time.time() + self._current_status = TestStatus.ABORTED + self._messages = [] + self._stacktrace = None + self._stdout = "" + self._stderr = "" + self._start_instr_count = self._instr_counter.read() + + def _find_next_test(self, suite): + """ + Find the next test that has not been run. + """ + + for test in suite: + + # We identify test suites by test that are iterable (as is done in + # the builtin python test harness). If we see one, recurse on it. + if hasattr(test, "__iter__"): + test = self._find_next_test(test) + + # The builtin python test harness sets test references to `None` + # after they have run, so we know we've found the next test up + # if it's not `None`. + if test is not None: + return test + + def stopTest(self, test): + sys.stdout = self._saved_stdout + sys.stderr = self._saved_stderr + + super(BuckTestResult, self).stopTest(test) + + # If a failure occured during module/class setup, then this "test" may + # actually be a `_ErrorHolder`, which doesn't contain explicit info + # about the upcoming test. Since we really only care about the test + # name field (i.e. `_testMethodName`), we use that to detect an actual + # test cases, and fall back to looking the test up from the suite + # otherwise. + if not hasattr(test, "_testMethodName"): + test = self._find_next_test(self._suite) + + result = { + "testCaseName": "{0}.{1}".format( + test.__class__.__module__, test.__class__.__name__ + ), + "testCase": test._testMethodName, + "type": self._current_status, + "time": int((time.time() - self._test_start_time) * 1000), + "message": os.linesep.join(self._messages), + "stacktrace": self._stacktrace, + "stdOut": self._stdout, + "stdErr": self._stderr, + } + + # TestPilot supports an instruction count field. + if "TEST_PILOT" in os.environ: + result["instrCount"] = ( + int(self._instr_counter.read() - self._start_instr_count), + ) + + self._results.append(result) + self._current_test = None + + def stopTestRun(self): + cov = self._main_program.get_coverage() + if cov is not None: + self._results.append({"coverage": cov}) + + @contextlib.contextmanager + def _withTest(self, test): + self.startTest(test) + yield + self.stopTest(test) + + def _setStatus(self, test, status, message=None, stacktrace=None): + assert test == self._current_test + self._current_status = status + self._stacktrace = stacktrace + if message is not None: + if message.endswith(os.linesep): + message = message[:-1] + self._messages.append(message) + + def setStatus(self, test, status, message=None, stacktrace=None): + # addError() may be called outside of a test if one of the shared + # fixtures (setUpClass/tearDownClass/setUpModule/tearDownModule) + # throws an error. + # + # In this case, create a fake test result to record the error. + if self._current_test is None: + with self._withTest(test): + self._setStatus(test, status, message, stacktrace) + else: + self._setStatus(test, status, message, stacktrace) + + def setException(self, test, status, excinfo): + exctype, value, tb = excinfo + self.setStatus( + test, + status, + "{0}: {1}".format(exctype.__name__, value), + "".join(traceback.format_tb(tb)), + ) + + def addSuccess(self, test): + super(BuckTestResult, self).addSuccess(test) + self.setStatus(test, TestStatus.PASSED) + + def addError(self, test, err): + super(BuckTestResult, self).addError(test, err) + self.setException(test, TestStatus.ABORTED, err) + + def addFailure(self, test, err): + super(BuckTestResult, self).addFailure(test, err) + self.setException(test, TestStatus.FAILED, err) + + def addSkip(self, test, reason): + super(BuckTestResult, self).addSkip(test, reason) + self.setStatus(test, TestStatus.SKIPPED, "Skipped: %s" % (reason,)) + + def addExpectedFailure(self, test, err): + super(BuckTestResult, self).addExpectedFailure(test, err) + self.setException(test, TestStatus.EXPECTED_FAILURE, err) + + def addUnexpectedSuccess(self, test): + super(BuckTestResult, self).addUnexpectedSuccess(test) + self.setStatus(test, TestStatus.UNEXPECTED_SUCCESS, "Unexpected success") + + def addStdout(self, val): + self._stdout += val + if self._show_output: + self._saved_stdout.write(val) + self._saved_stdout.flush() + + def addStdoutBytes(self, val): + string = val.decode("utf-8", errors="backslashreplace") + self.addStdout(string) + + def addStderr(self, val): + self._stderr += val + if self._show_output: + self._saved_stderr.write(val) + self._saved_stderr.flush() + + def addStderrBytes(self, val): + string = val.decode("utf-8", errors="backslashreplace") + self.addStderr(string) + + +class BuckTestRunner(unittest.TextTestRunner): + def __init__(self, main_program, suite, show_output=True, **kwargs): + super(BuckTestRunner, self).__init__(**kwargs) + self.show_output = show_output + self._main_program = main_program + self._suite = suite + + def _makeResult(self): + return BuckTestResult( + self.stream, + self.descriptions, + self.verbosity, + self.show_output, + self._main_program, + self._suite, + ) + + +def _format_test_name(test_class, attrname): + return "{0}.{1}.{2}".format(test_class.__module__, test_class.__name__, attrname) + + +class StderrLogHandler(logging.StreamHandler): + """ + This class is very similar to logging.StreamHandler, except that it + always uses the current sys.stderr object. + + StreamHandler caches the current sys.stderr object when it is constructed. + This makes it behave poorly in unit tests, which may replace sys.stderr + with a StringIO buffer during tests. The StreamHandler will continue using + the old sys.stderr object instead of the desired StringIO buffer. + """ + + def __init__(self): + logging.Handler.__init__(self) + + @property + def stream(self): + return sys.stderr + + +class RegexTestLoader(unittest.TestLoader): + def __init__(self, regex=None): + self.regex = regex + super(RegexTestLoader, self).__init__() + + def getTestCaseNames(self, testCaseClass): + """ + Return a sorted sequence of method names found within testCaseClass + """ + + testFnNames = super(RegexTestLoader, self).getTestCaseNames(testCaseClass) + if self.regex is None: + return testFnNames + robj = re.compile(self.regex) + matched = [] + for attrname in testFnNames: + fullname = _format_test_name(testCaseClass, attrname) + if robj.search(fullname): + matched.append(attrname) + return matched + + +class Loader(object): + + suiteClass = unittest.TestSuite + + def __init__(self, modules, regex=None): + self.modules = modules + self.regex = regex + + def load_all(self): + loader = RegexTestLoader(self.regex) + test_suite = self.suiteClass() + for module_name in self.modules: + __import__(module_name, level=0) + module = sys.modules[module_name] + module_suite = loader.loadTestsFromModule(module) + test_suite.addTest(module_suite) + return test_suite + + def load_args(self, args): + loader = RegexTestLoader(self.regex) + + suites = [] + for arg in args: + suite = loader.loadTestsFromName(arg) + # loadTestsFromName() can only process names that refer to + # individual test functions or modules. It can't process package + # names. If there were no module/function matches, check to see if + # this looks like a package name. + if suite.countTestCases() != 0: + suites.append(suite) + continue + + # Load all modules whose name is . + prefix = arg + "." + for module in self.modules: + if module.startswith(prefix): + suite = loader.loadTestsFromName(module) + suites.append(suite) + + return loader.suiteClass(suites) + + +_COVERAGE_INI = """\ +[report] +exclude_lines = + pragma: no cover + pragma: nocover + pragma:.*no${PLATFORM} + pragma:.*no${PY_IMPL}${PY_MAJOR}${PY_MINOR} + pragma:.*no${PY_IMPL}${PY_MAJOR} + pragma:.*nopy${PY_MAJOR} + pragma:.*nopy${PY_MAJOR}${PY_MINOR} +""" + + +class MainProgram(object): + """ + This class implements the main program. It can be subclassed by + users who wish to customize some parts of the main program. + (Adding additional command line options, customizing test loading, etc.) + """ + + DEFAULT_VERBOSITY = 2 + + def __init__(self, argv): + self.init_option_parser() + self.parse_options(argv) + self.setup_logging() + + def init_option_parser(self): + usage = "%prog [options] [TEST] ..." + op = optparse.OptionParser(usage=usage, add_help_option=False) + self.option_parser = op + + op.add_option( + "--hide-output", + dest="show_output", + action="store_false", + default=True, + help="Suppress data that tests print to stdout/stderr, and only " + "show it if the test fails.", + ) + op.add_option( + "-o", + "--output", + help="Write results to a file in a JSON format to be read by Buck", + ) + op.add_option( + "-f", + "--failfast", + action="store_true", + default=False, + help="Stop after the first failure", + ) + op.add_option( + "-l", + "--list-tests", + action="store_true", + dest="list", + default=False, + help="List tests and exit", + ) + op.add_option( + "-r", + "--regex", + default=None, + help="Regex to apply to tests, to only run those tests", + ) + op.add_option( + "--collect-coverage", + action="store_true", + default=False, + help="Collect test coverage information", + ) + op.add_option( + "--coverage-include", + default="*", + help='File globs to include in converage (split by ",")', + ) + op.add_option( + "--coverage-omit", + default="", + help='File globs to omit from converage (split by ",")', + ) + op.add_option( + "--logger", + action="append", + metavar="=", + default=[], + help="Configure log levels for specific logger categories", + ) + op.add_option( + "-q", + "--quiet", + action="count", + default=0, + help="Decrease the verbosity (may be specified multiple times)", + ) + op.add_option( + "-v", + "--verbosity", + action="count", + default=self.DEFAULT_VERBOSITY, + help="Increase the verbosity (may be specified multiple times)", + ) + op.add_option( + "-?", "--help", action="help", help="Show this help message and exit" + ) + + def parse_options(self, argv): + self.options, self.test_args = self.option_parser.parse_args(argv[1:]) + self.options.verbosity -= self.options.quiet + + if self.options.collect_coverage and coverage is None: + self.option_parser.error("coverage module is not available") + self.options.coverage_include = self.options.coverage_include.split(",") + if self.options.coverage_omit == "": + self.options.coverage_omit = [] + else: + self.options.coverage_omit = self.options.coverage_omit.split(",") + + def setup_logging(self): + # Configure the root logger to log at INFO level. + # This is similar to logging.basicConfig(), but uses our + # StderrLogHandler instead of a StreamHandler. + fmt = logging.Formatter("%(pathname)s:%(lineno)s: %(message)s") + log_handler = StderrLogHandler() + log_handler.setFormatter(fmt) + root_logger = logging.getLogger() + root_logger.addHandler(log_handler) + root_logger.setLevel(logging.INFO) + + level_names = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warn": logging.WARNING, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, + "fatal": logging.FATAL, + } + + for value in self.options.logger: + parts = value.rsplit("=", 1) + if len(parts) != 2: + self.option_parser.error( + "--logger argument must be of the " + "form =: %s" % value + ) + name = parts[0] + level_name = parts[1].lower() + level = level_names.get(level_name) + if level is None: + self.option_parser.error( + "invalid log level %r for log " "category %s" % (parts[1], name) + ) + logging.getLogger(name).setLevel(level) + + def create_loader(self): + import __test_modules__ + + return Loader(__test_modules__.TEST_MODULES, self.options.regex) + + def load_tests(self): + loader = self.create_loader() + if self.options.collect_coverage: + self.start_coverage() + include = self.options.coverage_include + omit = self.options.coverage_omit + if include and "*" not in include: + optimize_for_coverage(self.cov, include, omit) + + if self.test_args: + suite = loader.load_args(self.test_args) + else: + suite = loader.load_all() + if self.options.collect_coverage: + self.cov.start() + return suite + + def get_tests(self, test_suite): + tests = [] + + for test in test_suite: + if isinstance(test, unittest.TestSuite): + tests.extend(self.get_tests(test)) + else: + tests.append(test) + + return tests + + def run(self): + test_suite = self.load_tests() + + if self.options.list: + for test in self.get_tests(test_suite): + method_name = getattr(test, "_testMethodName", "") + name = _format_test_name(test.__class__, method_name) + print(name) + return EXIT_CODE_SUCCESS + else: + result = self.run_tests(test_suite) + if self.options.output is not None: + with open(self.options.output, "w") as f: + json.dump(result.getResults(), f, indent=4, sort_keys=True) + if not result.wasSuccessful(): + return EXIT_CODE_TEST_FAILURE + return EXIT_CODE_SUCCESS + + def run_tests(self, test_suite): + # Install a signal handler to catch Ctrl-C and display the results + # (but only if running >2.6). + if sys.version_info[0] > 2 or sys.version_info[1] > 6: + unittest.installHandler() + + # Run the tests + runner = BuckTestRunner( + self, + test_suite, + verbosity=self.options.verbosity, + show_output=self.options.show_output, + ) + result = runner.run(test_suite) + + if self.options.collect_coverage and self.options.show_output: + self.cov.stop() + try: + self.cov.report(file=sys.stdout) + except coverage.misc.CoverageException: + print("No lines were covered, potentially restricted by file filters") + + return result + + def get_abbr_impl(self): + """Return abbreviated implementation name.""" + impl = platform.python_implementation() + if impl == "PyPy": + return "pp" + elif impl == "Jython": + return "jy" + elif impl == "IronPython": + return "ip" + elif impl == "CPython": + return "cp" + else: + raise RuntimeError("unknown python runtime") + + def start_coverage(self): + if not self.options.collect_coverage: + return + + with tempfile.NamedTemporaryFile("w", delete=False) as coverage_ini: + coverage_ini.write(_COVERAGE_INI) + self._coverage_ini_path = coverage_ini.name + + # Keep the original working dir in case tests use os.chdir + self._original_working_dir = os.getcwd() + + # for coverage config ignores by platform/python version + os.environ["PLATFORM"] = sys.platform + os.environ["PY_IMPL"] = self.get_abbr_impl() + os.environ["PY_MAJOR"] = str(sys.version_info.major) + os.environ["PY_MINOR"] = str(sys.version_info.minor) + + self.cov = coverage.Coverage( + include=self.options.coverage_include, + omit=self.options.coverage_omit, + config_file=coverage_ini.name, + ) + self.cov.erase() + self.cov.start() + + def get_coverage(self): + if not self.options.collect_coverage: + return None + + try: + os.remove(self._coverage_ini_path) + except OSError: + pass # Better to litter than to fail the test + + # Switch back to the original working directory. + os.chdir(self._original_working_dir) + + result = {} + + self.cov.stop() + + try: + f = StringIO() + self.cov.report(file=f) + lines = f.getvalue().split("\n") + except coverage.misc.CoverageException: + # Nothing was covered. That's fine by us + return result + + # N.B.: the format of the coverage library's output differs + # depending on whether one or more files are in the results + for line in lines[2:]: + if line.strip("-") == "": + break + r = line.split()[0] + analysis = self.cov.analysis2(r) + covString = self.convert_to_diff_cov_str(analysis) + if covString: + result[r] = covString + + return result + + def convert_to_diff_cov_str(self, analysis): + # Info on the format of analysis: + # http://nedbatchelder.com/code/coverage/api.html + if not analysis: + return None + numLines = max( + analysis[1][-1] if len(analysis[1]) else 0, + analysis[2][-1] if len(analysis[2]) else 0, + analysis[3][-1] if len(analysis[3]) else 0, + ) + lines = ["N"] * numLines + for l in analysis[1]: + lines[l - 1] = "C" + for l in analysis[2]: + lines[l - 1] = "X" + for l in analysis[3]: + lines[l - 1] = "U" + return "".join(lines) + + +def main(argv): + return MainProgram(sys.argv).run() + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build/fbcode_builder/CMake/fb_py_win_main.c b/build/fbcode_builder/CMake/fb_py_win_main.c new file mode 100644 index 000000000000..8905c36025bd --- /dev/null +++ b/build/fbcode_builder/CMake/fb_py_win_main.c @@ -0,0 +1,126 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +#define WIN32_LEAN_AND_MEAN + +#include +#include +#include + +#define PATH_SIZE 32768 + +typedef int (*Py_Main)(int, wchar_t**); + +// Add the given path to Windows's DLL search path. +// For Windows DLL search path resolution, see: +// https://docs.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order +void add_search_path(const wchar_t* path) { + wchar_t buffer[PATH_SIZE]; + wchar_t** lppPart = NULL; + + if (!GetFullPathNameW(path, PATH_SIZE, buffer, lppPart)) { + fwprintf(stderr, L"warning: %d unable to expand path %s\n", GetLastError(), path); + return; + } + + if (!AddDllDirectory(buffer)) { + DWORD error = GetLastError(); + if (error != ERROR_FILE_NOT_FOUND) { + fwprintf(stderr, L"warning: %d unable to set DLL search path for %s\n", GetLastError(), path); + } + } +} + +int locate_py_main(int argc, wchar_t **argv) { + /* + * We have to dynamically locate Python3.dll because we may be loading a + * Python native module while running. If that module is built with a + * different Python version, we will end up a DLL import error. To resolve + * this, we can either ship an embedded version of Python with us or + * dynamically look up existing Python distribution installed on user's + * machine. This way, we should be able to get a consistent version of + * Python3.dll and .pyd modules. + */ + HINSTANCE python_dll; + Py_Main pymain; + + // last added directory has highest priority + add_search_path(L"C:\\Python36\\"); + add_search_path(L"C:\\Python37\\"); + add_search_path(L"C:\\Python38\\"); + + python_dll = LoadLibraryExW(L"python3.dll", NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + + int returncode = 0; + if (python_dll != NULL) { + pymain = (Py_Main) GetProcAddress(python_dll, "Py_Main"); + + if (pymain != NULL) { + returncode = (pymain)(argc, argv); + } else { + fprintf(stderr, "error: %d unable to load Py_Main\n", GetLastError()); + } + + FreeLibrary(python_dll); + } else { + fprintf(stderr, "error: %d unable to locate python3.dll\n", GetLastError()); + return 1; + } + return returncode; +} + +int wmain() { + /* + * This executable will be prepended to the start of a Python ZIP archive. + * Python will be able to directly execute the ZIP archive, so we simply + * need to tell Py_Main() to run our own file. Duplicate the argument list + * and add our file name to the beginning to tell Python what file to invoke. + */ + wchar_t** pyargv = malloc(sizeof(wchar_t*) * (__argc + 1)); + if (!pyargv) { + fprintf(stderr, "error: failed to allocate argument vector\n"); + return 1; + } + + /* Py_Main wants the wide character version of the argv so we pull those + * values from the global __wargv array that has been prepared by MSVCRT. + * + * In order for the zipapp to run we need to insert an extra argument in + * the front of the argument vector that points to ourselves. + * + * An additional complication is that, depending on who prepared the argument + * string used to start our process, the computed __wargv[0] can be a simple + * shell word like `watchman-wait` which is normally resolved together with + * the PATH by the shell. + * That unresolved path isn't sufficient to start the zipapp on windows; + * we need the fully qualified path. + * + * Given: + * __wargv == {"watchman-wait", "-h"} + * + * we want to pass the following to Py_Main: + * + * { + * "z:\build\watchman\python\watchman-wait.exe", + * "z:\build\watchman\python\watchman-wait.exe", + * "-h" + * } + */ + wchar_t full_path_to_argv0[PATH_SIZE]; + DWORD len = GetModuleFileNameW(NULL, full_path_to_argv0, PATH_SIZE); + if (len == 0 || + len == PATH_SIZE && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + fprintf( + stderr, + "error: %d while retrieving full path to this executable\n", + GetLastError()); + return 1; + } + + for (int n = 1; n < __argc; ++n) { + pyargv[n + 1] = __wargv[n]; + } + pyargv[0] = full_path_to_argv0; + pyargv[1] = full_path_to_argv0; + + return locate_py_main(__argc + 1, pyargv); +} diff --git a/build/fbcode_builder/CMake/make_fbpy_archive.py b/build/fbcode_builder/CMake/make_fbpy_archive.py new file mode 100755 index 000000000000..3724feb2183f --- /dev/null +++ b/build/fbcode_builder/CMake/make_fbpy_archive.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +# +# Copyright (c) Facebook, Inc. and its affiliates. +# +import argparse +import collections +import errno +import os +import shutil +import sys +import tempfile +import zipapp + +MANIFEST_SEPARATOR = " :: " +MANIFEST_HEADER_V1 = "FBPY_MANIFEST 1\n" + + +class UsageError(Exception): + def __init__(self, message): + self.message = message + + def __str__(self): + return self.message + + +class BadManifestError(UsageError): + def __init__(self, path, line_num, message): + full_msg = "%s:%s: %s" % (path, line_num, message) + super().__init__(full_msg) + self.path = path + self.line_num = line_num + self.raw_message = message + + +PathInfo = collections.namedtuple( + "PathInfo", ("src", "dest", "manifest_path", "manifest_line") +) + + +def parse_manifest(manifest, path_map): + bad_prefix = ".." + os.path.sep + manifest_dir = os.path.dirname(manifest) + with open(manifest, "r") as f: + line_num = 1 + line = f.readline() + if line != MANIFEST_HEADER_V1: + raise BadManifestError( + manifest, line_num, "Unexpected manifest file header" + ) + + for line in f: + line_num += 1 + if line.startswith("#"): + continue + line = line.rstrip("\n") + parts = line.split(MANIFEST_SEPARATOR) + if len(parts) != 2: + msg = "line must be of the form SRC %s DEST" % MANIFEST_SEPARATOR + raise BadManifestError(manifest, line_num, msg) + src, dest = parts + dest = os.path.normpath(dest) + if dest.startswith(bad_prefix): + msg = "destination path starts with %s: %s" % (bad_prefix, dest) + raise BadManifestError(manifest, line_num, msg) + + if not os.path.isabs(src): + src = os.path.normpath(os.path.join(manifest_dir, src)) + + if dest in path_map: + prev_info = path_map[dest] + msg = ( + "multiple source paths specified for destination " + "path %s. Previous source was %s from %s:%s" + % ( + dest, + prev_info.src, + prev_info.manifest_path, + prev_info.manifest_line, + ) + ) + raise BadManifestError(manifest, line_num, msg) + + info = PathInfo( + src=src, + dest=dest, + manifest_path=manifest, + manifest_line=line_num, + ) + path_map[dest] = info + + +def populate_install_tree(inst_dir, path_map): + os.mkdir(inst_dir) + dest_dirs = {"": False} + + def make_dest_dir(path): + if path in dest_dirs: + return + parent = os.path.dirname(path) + make_dest_dir(parent) + abs_path = os.path.join(inst_dir, path) + os.mkdir(abs_path) + dest_dirs[path] = False + + def install_file(info): + dir_name, base_name = os.path.split(info.dest) + make_dest_dir(dir_name) + if base_name == "__init__.py": + dest_dirs[dir_name] = True + abs_dest = os.path.join(inst_dir, info.dest) + shutil.copy2(info.src, abs_dest) + + # Copy all of the destination files + for info in path_map.values(): + install_file(info) + + # Create __init__ files in any directories that don't have them. + for dir_path, has_init in dest_dirs.items(): + if has_init: + continue + init_path = os.path.join(inst_dir, dir_path, "__init__.py") + with open(init_path, "w"): + pass + + +def build_zipapp(args, path_map): + """Create a self executing python binary using Python 3's built-in + zipapp module. + + This type of Python binary is relatively simple, as zipapp is part of the + standard library, but it does not support native language extensions + (.so/.dll files). + """ + dest_dir = os.path.dirname(args.output) + with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir: + inst_dir = os.path.join(tmpdir, "tree") + populate_install_tree(inst_dir, path_map) + + tmp_output = os.path.join(tmpdir, "output.exe") + zipapp.create_archive( + inst_dir, target=tmp_output, interpreter=args.python, main=args.main + ) + os.replace(tmp_output, args.output) + + +def create_main_module(args, inst_dir, path_map): + if not args.main: + assert "__main__.py" in path_map + return + + dest_path = os.path.join(inst_dir, "__main__.py") + main_module, main_fn = args.main.split(":") + main_contents = """\ +#!{python} + +if __name__ == "__main__": + import {main_module} + {main_module}.{main_fn}() +""".format( + python=args.python, main_module=main_module, main_fn=main_fn + ) + with open(dest_path, "w") as f: + f.write(main_contents) + os.chmod(dest_path, 0o755) + + +def build_install_dir(args, path_map): + """Create a directory that contains all of the sources, with a __main__ + module to run the program. + """ + # Populate a temporary directory first, then rename to the destination + # location. This ensures that we don't ever leave a halfway-built + # directory behind at the output path if something goes wrong. + dest_dir = os.path.dirname(args.output) + with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir: + inst_dir = os.path.join(tmpdir, "tree") + populate_install_tree(inst_dir, path_map) + create_main_module(args, inst_dir, path_map) + os.rename(inst_dir, args.output) + + +def ensure_directory(path): + try: + os.makedirs(path) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + + +def install_library(args, path_map): + """Create an installation directory a python library.""" + out_dir = args.output + out_manifest = args.output + ".manifest" + + install_dir = args.install_dir + if not install_dir: + install_dir = out_dir + + os.makedirs(out_dir) + with open(out_manifest, "w") as manifest: + manifest.write(MANIFEST_HEADER_V1) + for info in path_map.values(): + abs_dest = os.path.join(out_dir, info.dest) + ensure_directory(os.path.dirname(abs_dest)) + print("copy %r --> %r" % (info.src, abs_dest)) + shutil.copy2(info.src, abs_dest) + installed_dest = os.path.join(install_dir, info.dest) + manifest.write("%s%s%s\n" % (installed_dest, MANIFEST_SEPARATOR, info.dest)) + + +def parse_manifests(args): + # Process args.manifest_separator to help support older versions of CMake + if args.manifest_separator: + manifests = [] + for manifest_arg in args.manifests: + split_arg = manifest_arg.split(args.manifest_separator) + manifests.extend(split_arg) + args.manifests = manifests + + path_map = {} + for manifest in args.manifests: + parse_manifest(manifest, path_map) + + return path_map + + +def check_main_module(args, path_map): + # Translate an empty string in the --main argument to None, + # just to allow the CMake logic to be slightly simpler and pass in an + # empty string when it really wants the default __main__.py module to be + # used. + if args.main == "": + args.main = None + + if args.type == "lib-install": + if args.main is not None: + raise UsageError("cannot specify a --main argument with --type=lib-install") + return + + main_info = path_map.get("__main__.py") + if args.main: + if main_info is not None: + msg = ( + "specified an explicit main module with --main, " + "but the file listing already includes __main__.py" + ) + raise BadManifestError( + main_info.manifest_path, main_info.manifest_line, msg + ) + parts = args.main.split(":") + if len(parts) != 2: + raise UsageError( + "argument to --main must be of the form MODULE:CALLABLE " + "(received %s)" % (args.main,) + ) + else: + if main_info is None: + raise UsageError( + "no main module specified with --main, " + "and no __main__.py module present" + ) + + +BUILD_TYPES = { + "zipapp": build_zipapp, + "dir": build_install_dir, + "lib-install": install_library, +} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("-o", "--output", required=True, help="The output file path") + ap.add_argument( + "--install-dir", + help="When used with --type=lib-install, this parameter specifies the " + "final location where the library where be installed. This can be " + "used to generate the library in one directory first, when you plan " + "to move or copy it to another final location later.", + ) + ap.add_argument( + "--manifest-separator", + help="Split manifest arguments around this separator. This is used " + "to support older versions of CMake that cannot supply the manifests " + "as separate arguments.", + ) + ap.add_argument( + "--main", + help="The main module to run, specified as :. " + "This must be specified if and only if the archive does not contain " + "a __main__.py file.", + ) + ap.add_argument( + "--python", + help="Explicitly specify the python interpreter to use for the " "executable.", + ) + ap.add_argument( + "--type", choices=BUILD_TYPES.keys(), help="The type of output to build." + ) + ap.add_argument( + "manifests", + nargs="+", + help="The manifest files specifying how to construct the archive", + ) + args = ap.parse_args() + + if args.python is None: + args.python = sys.executable + + if args.type is None: + # In the future we might want different default output types + # for different platforms. + args.type = "zipapp" + build_fn = BUILD_TYPES[args.type] + + try: + path_map = parse_manifests(args) + check_main_module(args, path_map) + except UsageError as ex: + print("error: %s" % (ex,), file=sys.stderr) + sys.exit(1) + + build_fn(args, path_map) + + +if __name__ == "__main__": + main() diff --git a/build/fbcode_builder/LICENSE b/build/fbcode_builder/LICENSE new file mode 100644 index 000000000000..b96dcb0480a0 --- /dev/null +++ b/build/fbcode_builder/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Facebook, Inc. and its affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/build/fbcode_builder/README.docker b/build/fbcode_builder/README.docker new file mode 100644 index 000000000000..4e9fa8a2941c --- /dev/null +++ b/build/fbcode_builder/README.docker @@ -0,0 +1,44 @@ +## Debugging Docker builds + +To debug a a build failure, start up a shell inside the just-failed image as +follows: + +``` +docker ps -a | head # Grab the container ID +docker commit CONTAINER_ID # Grab the SHA string +docker run -it SHA_STRING /bin/bash +# Debug as usual, e.g. `./run-cmake.sh Debug`, `make`, `apt-get install gdb` +``` + +## A note on Docker security + +While the Dockerfile generated above is quite simple, you must be aware that +using Docker to run arbitrary code can present significant security risks: + + - Code signature validation is off by default (as of 2016), exposing you to + man-in-the-middle malicious code injection. + + - You implicitly trust the world -- a Dockerfile cannot annotate that + you trust the image `debian:8.6` because you trust a particular + certificate -- rather, you trust the name, and that it will never be + hijacked. + + - Sandboxing in the Linux kernel is not perfect, and the builds run code as + root. Any compromised code can likely escalate to the host system. + +Specifically, you must be very careful only to add trusted OS images to the +build flow. + +Consider setting this variable before running any Docker container -- this +will validate a signature on the base image before running code from it: + +``` +export DOCKER_CONTENT_TRUST=1 +``` + +Note that unless you go through the extra steps of notarizing the resulting +images, you will have to disable trust to enter intermediate images, e.g. + +``` +DOCKER_CONTENT_TRUST= docker run -it YOUR_IMAGE_ID /bin/bash +``` diff --git a/build/fbcode_builder/README.md b/build/fbcode_builder/README.md new file mode 100644 index 000000000000..d47dd41c0149 --- /dev/null +++ b/build/fbcode_builder/README.md @@ -0,0 +1,43 @@ +# Easy builds for Facebook projects + +This directory contains tools designed to simplify continuous-integration +(and other builds) of Facebook open source projects. In particular, this helps +manage builds for cross-project dependencies. + +The main entry point is the `getdeps.py` script. This script has several +subcommands, but the most notable is the `build` command. This will download +and build all dependencies for a project, and then build the project itself. + +## Deployment + +This directory is copied literally into a number of different Facebook open +source repositories. Any change made to code in this directory will be +automatically be replicated by our open source tooling into all GitHub hosted +repositories that use `fbcode_builder`. Typically this directory is copied +into the open source repositories as `build/fbcode_builder/`. + + +# Project Configuration Files + +The `manifests` subdirectory contains configuration files for many different +projects, describing how to build each project. These files also list +dependencies between projects, enabling `getdeps.py` to build all dependencies +for a project before building the project itself. + + +# Shared CMake utilities + +Since this directory is copied into many Facebook open source repositories, +it is also used to help share some CMake utility files across projects. The +`CMake/` subdirectory contains a number of `.cmake` files that are shared by +the CMake-based build systems across several different projects. + + +# Older Build Scripts + +This directory also still contains a handful of older build scripts that +pre-date the current `getdeps.py` build system. Most of the other `.py` files +in this top directory, apart from `getdeps.py` itself, are from this older +build system. This older system is only used by a few remaining projects, and +new projects should generally use the newer `getdeps.py` script, by adding a +new configuration file in the `manifests/` subdirectory. diff --git a/build/fbcode_builder/docker_build_with_ccache.sh b/build/fbcode_builder/docker_build_with_ccache.sh new file mode 100755 index 000000000000..e922810d59a3 --- /dev/null +++ b/build/fbcode_builder/docker_build_with_ccache.sh @@ -0,0 +1,219 @@ +#!/bin/bash -uex +# Copyright (c) Facebook, Inc. and its affiliates. +set -o pipefail # Be sure to `|| :` commands that are allowed to fail. + +# +# Future: port this to Python if you are making significant changes. +# + +# Parse command-line arguments +build_timeout="" # Default to no time-out +print_usage() { + echo "Usage: $0 [--build-timeout TIMEOUT_VAL] SAVE-CCACHE-TO-DIR" + echo "SAVE-CCACHE-TO-DIR is required. An empty string discards the ccache." +} +while [[ $# -gt 0 ]]; do + case "$1" in + --build-timeout) + shift + build_timeout="$1" + if [[ "$build_timeout" != "" ]] ; then + timeout "$build_timeout" true # fail early on invalid timeouts + fi + ;; + -h|--help) + print_usage + exit + ;; + *) + break + ;; + esac + shift +done +# There is one required argument, but an empty string is allowed. +if [[ "$#" != 1 ]] ; then + print_usage + exit 1 +fi +save_ccache_to_dir="$1" +if [[ "$save_ccache_to_dir" != "" ]] ; then + mkdir -p "$save_ccache_to_dir" # fail early if there's nowhere to save +else + echo "WARNING: Will not save /ccache from inside the Docker container" +fi + +rand_guid() { + echo "$(date +%s)_${RANDOM}_${RANDOM}_${RANDOM}_${RANDOM}" +} + +id=fbcode_builder_image_id=$(rand_guid) +logfile=$(mktemp) + +echo " + + +Running build with timeout '$build_timeout', label $id, and log in $logfile + + +" + +if [[ "$build_timeout" != "" ]] ; then + # Kill the container after $build_timeout. Using `/bin/timeout` would cause + # Docker to destroy the most recent container and lose its cache. + ( + sleep "$build_timeout" + echo "Build timed out after $build_timeout" 1>&2 + while true; do + maybe_container=$( + grep -E '^( ---> Running in [0-9a-f]+|FBCODE_BUILDER_EXIT)$' "$logfile" | + tail -n 1 | awk '{print $NF}' + ) + if [[ "$maybe_container" == "FBCODE_BUILDER_EXIT" ]] ; then + echo "Time-out successfully terminated build" 1>&2 + break + fi + echo "Time-out: trying to kill $maybe_container" 1>&2 + # This kill fail if we get unlucky, try again soon. + docker kill "$maybe_container" || sleep 5 + done + ) & +fi + +build_exit_code=0 +# `docker build` is allowed to fail, and `pipefail` means we must check the +# failure explicitly. +if ! docker build --label="$id" . 2>&1 | tee "$logfile" ; then + build_exit_code="${PIPESTATUS[0]}" + # NB: We are going to deliberately forge ahead even if `tee` failed. + # If it did, we have a problem with tempfile creation, and all is sad. + echo "Build failed with code $build_exit_code, trying to save ccache" 1>&2 +fi +# Stop trying to kill the container. +echo $'\nFBCODE_BUILDER_EXIT' >> "$logfile" + +if [[ "$save_ccache_to_dir" == "" ]] ; then + echo "Not inspecting Docker build, since saving the ccache wasn't requested." + exit "$build_exit_code" +fi + +img=$(docker images --filter "label=$id" -a -q) +if [[ "$img" == "" ]] ; then + docker images -a + echo "In the above list, failed to find most recent image with $id" 1>&2 + # Usually, the above `docker kill` will leave us with an up-to-the-second + # container, from which we can extract the cache. However, if that fails + # for any reason, this loop will instead grab the latest available image. + # + # It's possible for this log search to get confused due to the output of + # the build command itself, but since our builds aren't **trying** to + # break cache, we probably won't randomly hit an ID from another build. + img=$( + grep -E '^ ---> (Running in [0-9a-f]+|[0-9a-f]+)$' "$logfile" | tac | + sed 's/Running in /container_/;s/ ---> //;' | ( + while read -r x ; do + # Both docker commands below print an image ID to stdout on + # success, so we just need to know when to stop. + if [[ "$x" =~ container_.* ]] ; then + if docker commit "${x#container_}" ; then + break + fi + elif docker inspect --type image -f '{{.Id}}' "$x" ; then + break + fi + done + ) + ) + if [[ "$img" == "" ]] ; then + echo "Failed to find valid container or image ID in log $logfile" 1>&2 + exit 1 + fi +elif [[ "$(echo "$img" | wc -l)" != 1 ]] ; then + # Shouldn't really happen, but be explicit if it does. + echo "Multiple images with label $id, taking the latest of:" + echo "$img" + img=$(echo "$img" | head -n 1) +fi + +container_name="fbcode_builder_container_$(rand_guid)" +echo "Starting $container_name from latest image of the build with $id --" +echo "$img" + +# ccache collection must be done outside of the Docker build steps because +# we need to be able to kill it on timeout. +# +# This step grows the max cache size to slightly exceed than the working set +# of a successful build. This simple design persists the max size in the +# cache directory itself (the env var CCACHE_MAXSIZE does not even work with +# older ccaches like the one on 14.04). +# +# Future: copy this script into the Docker image via Dockerfile. +( + # By default, fbcode_builder creates an unsigned image, so the `docker + # run` below would fail if DOCKER_CONTENT_TRUST were set. So we unset it + # just for this one run. + export DOCKER_CONTENT_TRUST= + # CAUTION: The inner bash runs without -uex, so code accordingly. + docker run --user root --name "$container_name" "$img" /bin/bash -c ' + build_exit_code='"$build_exit_code"' + + # Might be useful if debugging whether max cache size is too small? + grep " Cleaning up cache directory " /tmp/ccache.log + + export CCACHE_DIR=/ccache + ccache -s + + echo "Total bytes in /ccache:"; + total_bytes=$(du -sb /ccache | awk "{print \$1}") + echo "$total_bytes" + + echo "Used bytes in /ccache:"; + used_bytes=$( + du -sb $(find /ccache -type f -newermt @$( + cat /FBCODE_BUILDER_CCACHE_START_TIME + )) | awk "{t += \$1} END {print t}" + ) + echo "$used_bytes" + + # Goal: set the max cache to 750MB over 125% of the usage of a + # successful build. If this is too small, it takes too long to get a + # cache fully warmed up. Plus, ccache cleans 100-200MB before reaching + # the max cache size, so a large margin is essential to prevent misses. + desired_mb=$(( 750 + used_bytes / 800000 )) # 125% in decimal MB: 1e6/1.25 + if [[ "$build_exit_code" != "0" ]] ; then + # For a bad build, disallow shrinking the max cache size. Instead of + # the max cache size, we use on-disk size, which ccache keeps at least + # 150MB under the actual max size, hence the 400MB safety margin. + cur_max_mb=$(( 400 + total_bytes / 1000000 )) # ccache uses decimal MB + if [[ "$desired_mb" -le "$cur_max_mb" ]] ; then + desired_mb="" + fi + fi + + if [[ "$desired_mb" != "" ]] ; then + echo "Updating cache size to $desired_mb MB" + ccache -M "${desired_mb}M" + ccache -s + fi + + # Subshell because `time` the binary may not be installed. + if (time tar czf /ccache.tgz /ccache) ; then + ls -l /ccache.tgz + else + # This `else` ensures we never overwrite the current cache with + # partial data in case of error, even if somebody adds code below. + rm /ccache.tgz + exit 1 + fi + ' +) + +echo "Updating $save_ccache_to_dir/ccache.tgz" +# This will not delete the existing cache if `docker run` didn't make one +docker cp "$container_name:/ccache.tgz" "$save_ccache_to_dir/" + +# Future: it'd be nice if Travis allowed us to retry if the build timed out, +# since we'll make more progress thanks to the cache. As-is, we have to +# wait for the next commit to land. +echo "Build exited with code $build_exit_code" +exit "$build_exit_code" diff --git a/build/fbcode_builder/docker_builder.py b/build/fbcode_builder/docker_builder.py new file mode 100644 index 000000000000..83df7137cf97 --- /dev/null +++ b/build/fbcode_builder/docker_builder.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +""" + +Extends FBCodeBuilder to produce Docker context directories. + +In order to get the largest iteration-time savings from Docker's build +caching, you will want to: + - Use fine-grained steps as appropriate (e.g. separate make & make install), + - Start your action sequence with the lowest-risk steps, and with the steps + that change the least often, and + - Put the steps that you are debugging towards the very end. + +""" +import logging +import os +import shutil +import tempfile + +from fbcode_builder import FBCodeBuilder +from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted, path_join +from utils import recursively_flatten_list, run_command + + +class DockerFBCodeBuilder(FBCodeBuilder): + def _user(self): + return self.option("user", "root") + + def _change_user(self): + return ShellQuoted("USER {u}").format(u=self._user()) + + def setup(self): + # Please add RPM-based OSes here as appropriate. + # + # To allow exercising non-root installs -- we change users after the + # system packages are installed. TODO: For users not defined in the + # image, we should probably `useradd`. + return self.step( + "Setup", + [ + # Docker's FROM does not understand shell quoting. + ShellQuoted("FROM {}".format(self.option("os_image"))), + # /bin/sh syntax is a pain + ShellQuoted('SHELL ["/bin/bash", "-c"]'), + ] + + self.install_debian_deps() + + [self._change_user()] + + [self.workdir(self.option("prefix"))] + + self.create_python_venv() + + self.python_venv() + + self.rust_toolchain(), + ) + + def python_venv(self): + # To both avoid calling venv activate on each RUN command AND to ensure + # it is present when the resulting container is run add to PATH + actions = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + actions = ShellQuoted("ENV PATH={p}:$PATH").format( + p=path_join(self.option("prefix"), "venv", "bin") + ) + return actions + + def step(self, name, actions): + assert "\n" not in name, "Name {0} would span > 1 line".format(name) + b = ShellQuoted("") + return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b] + + def run(self, shell_cmd): + return ShellQuoted("RUN {cmd}").format(cmd=shell_cmd) + + def set_env(self, key, value): + return ShellQuoted("ENV {key}={val}").format(key=key, val=value) + + def workdir(self, dir): + return [ + # As late as Docker 1.12.5, this results in `build` being owned + # by root:root -- the explicit `mkdir` works around the bug: + # USER nobody + # WORKDIR build + ShellQuoted("USER root"), + ShellQuoted("RUN mkdir -p {d} && chown {u} {d}").format( + d=dir, u=self._user() + ), + self._change_user(), + ShellQuoted("WORKDIR {dir}").format(dir=dir), + ] + + def comment(self, comment): + # This should not be a command since we don't want comment changes + # to invalidate the Docker build cache. + return shell_comment(comment) + + def copy_local_repo(self, repo_dir, dest_name): + fd, archive_path = tempfile.mkstemp( + prefix="local_repo_{0}_".format(dest_name), + suffix=".tgz", + dir=os.path.abspath(self.option("docker_context_dir")), + ) + os.close(fd) + run_command("tar", "czf", archive_path, ".", cwd=repo_dir) + return [ + ShellQuoted("ADD {archive} {dest_name}").format( + archive=os.path.basename(archive_path), dest_name=dest_name + ), + # Docker permissions make very little sense... see also workdir() + ShellQuoted("USER root"), + ShellQuoted("RUN chown -R {u} {d}").format(d=dest_name, u=self._user()), + self._change_user(), + ] + + def _render_impl(self, steps): + return raw_shell(shell_join("\n", recursively_flatten_list(steps))) + + def debian_ccache_setup_steps(self): + source_ccache_tgz = self.option("ccache_tgz", "") + if not source_ccache_tgz: + logging.info("Docker ccache not enabled") + return [] + + dest_ccache_tgz = os.path.join(self.option("docker_context_dir"), "ccache.tgz") + + try: + try: + os.link(source_ccache_tgz, dest_ccache_tgz) + except OSError: + logging.exception( + "Hard-linking {s} to {d} failed, falling back to copy".format( + s=source_ccache_tgz, d=dest_ccache_tgz + ) + ) + shutil.copyfile(source_ccache_tgz, dest_ccache_tgz) + except Exception: + logging.exception( + "Failed to copy or link {s} to {d}, aborting".format( + s=source_ccache_tgz, d=dest_ccache_tgz + ) + ) + raise + + return [ + # Separate layer so that in development we avoid re-downloads. + self.run(ShellQuoted("apt-get install -yq ccache")), + ShellQuoted("ADD ccache.tgz /"), + ShellQuoted( + # Set CCACHE_DIR before the `ccache` invocations below. + "ENV CCACHE_DIR=/ccache " + # No clang support for now, so it's easiest to hardcode gcc. + 'CC="ccache gcc" CXX="ccache g++" ' + # Always log for ease of debugging. For real FB projects, + # this log is several megabytes, so dumping it to stdout + # would likely exceed the Travis log limit of 4MB. + # + # On a local machine, `docker cp` will get you the data. To + # get the data out from Travis, I would compress and dump + # uuencoded bytes to the log -- for Bistro this was about + # 600kb or 8000 lines: + # + # apt-get install sharutils + # bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2 + "CCACHE_LOGFILE=/tmp/ccache.log" + ), + self.run( + ShellQuoted( + # Future: Skipping this part made this Docker step instant, + # saving ~1min of build time. It's unclear if it is the + # chown or the du, but probably the chown -- since a large + # part of the cost is incurred at image save time. + # + # ccache.tgz may be empty, or may have the wrong + # permissions. + "mkdir -p /ccache && time chown -R nobody /ccache && " + "time du -sh /ccache && " + # Reset stats so `docker_build_with_ccache.sh` can print + # useful values at the end of the run. + "echo === Prev run stats === && ccache -s && ccache -z && " + # Record the current time to let travis_build.sh figure out + # the number of bytes in the cache that are actually used -- + # this is crucial for tuning the maximum cache size. + "date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && " + # The build running as `nobody` should be able to write here + "chown nobody /tmp/ccache.log" + ) + ), + ] diff --git a/build/fbcode_builder/docker_enable_ipv6.sh b/build/fbcode_builder/docker_enable_ipv6.sh new file mode 100755 index 000000000000..3752f6f5e6ef --- /dev/null +++ b/build/fbcode_builder/docker_enable_ipv6.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. + + +# `daemon.json` is normally missing, but let's log it in case that changes. +touch /etc/docker/daemon.json +service docker stop +echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64"}' > /etc/docker/daemon.json +service docker start +# Fail early if docker failed on start -- add `- sudo dockerd` to debug. +docker info +# Paranoia log: what if our config got overwritten? +cat /etc/docker/daemon.json diff --git a/build/fbcode_builder/fbcode_builder.py b/build/fbcode_builder/fbcode_builder.py new file mode 100644 index 000000000000..742099321698 --- /dev/null +++ b/build/fbcode_builder/fbcode_builder.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +""" + +This is a small DSL to describe builds of Facebook's open-source projects +that are published to Github from a single internal repo, including projects +that depend on folly, wangle, proxygen, fbthrift, etc. + +This file defines the interface of the DSL, and common utilieis, but you +will have to instantiate a specific builder, with specific options, in +order to get work done -- see e.g. make_docker_context.py. + +== Design notes == + +Goals: + + - A simple declarative language for what needs to be checked out & built, + how, in what order. + + - The same specification should work for external continuous integration + builds (e.g. Travis + Docker) and for internal VM-based continuous + integration builds. + + - One should be able to build without root, and to install to a prefix. + +Non-goals: + + - General usefulness. The only point of this is to make it easier to build + and test Facebook's open-source services. + +Ideas for the future -- these may not be very good :) + + - Especially on Ubuntu 14.04 the current initial setup is inefficient: + we add PPAs after having installed a bunch of packages -- this prompts + reinstalls of large amounts of code. We also `apt-get update` a few + times. + + - A "shell script" builder. Like DockerFBCodeBuilder, but outputs a + shell script that runs outside of a container. Or maybe even + synchronously executes the shell commands, `make`-style. + + - A "Makefile" generator. That might make iterating on builds even quicker + than what you can currently get with Docker build caching. + + - Generate a rebuild script that can be run e.g. inside the built Docker + container by tagging certain steps with list-inheriting Python objects: + * do change directories + * do NOT `git clone` -- if we want to update code this should be a + separate script that e.g. runs rebase on top of specific targets + across all the repos. + * do NOT install software (most / all setup can be skipped) + * do NOT `autoreconf` or `configure` + * do `make` and `cmake` + + - If we get non-Debian OSes, part of ccache setup should be factored out. +""" + +import os +import re + +from shell_quoting import path_join, shell_join, ShellQuoted + + +def _read_project_github_hashes(): + base_dir = "deps/github_hashes/" # trailing slash used in regex below + for dirname, _, files in os.walk(base_dir): + for filename in files: + path = os.path.join(dirname, filename) + with open(path) as f: + m_proj = re.match("^" + base_dir + "(.*)-rev\.txt$", path) + if m_proj is None: + raise RuntimeError("Not a hash file? {0}".format(path)) + m_hash = re.match("^Subproject commit ([0-9a-f]+)\n$", f.read()) + if m_hash is None: + raise RuntimeError("No hash in {0}".format(path)) + yield m_proj.group(1), m_hash.group(1) + + +class FBCodeBuilder(object): + def __init__(self, **kwargs): + self._options_do_not_access = kwargs # Use .option() instead. + # This raises upon detecting options that are specified but unused, + # because otherwise it is very easy to make a typo in option names. + self.options_used = set() + # Mark 'projects_dir' used even if the build installs no github + # projects. This is needed because driver programs like + # `shell_builder.py` unconditionally set this for all builds. + self._github_dir = self.option("projects_dir") + self._github_hashes = dict(_read_project_github_hashes()) + + def __repr__(self): + return "{0}({1})".format( + self.__class__.__name__, + ", ".join( + "{0}={1}".format(k, repr(v)) + for k, v in self._options_do_not_access.items() + ), + ) + + def option(self, name, default=None): + value = self._options_do_not_access.get(name, default) + if value is None: + raise RuntimeError("Option {0} is required".format(name)) + self.options_used.add(name) + return value + + def has_option(self, name): + return name in self._options_do_not_access + + def add_option(self, name, value): + if name in self._options_do_not_access: + raise RuntimeError("Option {0} already set".format(name)) + self._options_do_not_access[name] = value + + # + # Abstract parts common to every installation flow + # + + def render(self, steps): + """ + + Converts nested actions to your builder's expected output format. + Typically takes the output of build(). + + """ + res = self._render_impl(steps) # Implementation-dependent + # Now that the output is rendered, we expect all options to have + # been used. + unused_options = set(self._options_do_not_access) + unused_options -= self.options_used + if unused_options: + raise RuntimeError( + "Unused options: {0} -- please check if you made a typo " + "in any of them. Those that are truly not useful should " + "be not be set so that this typo detection can be useful.".format( + unused_options + ) + ) + return res + + def build(self, steps): + if not steps: + raise RuntimeError( + "Please ensure that the config you are passing " "contains steps" + ) + return [self.setup(), self.diagnostics()] + steps + + def setup(self): + "Your builder may want to install packages here." + raise NotImplementedError + + def diagnostics(self): + "Log some system diagnostics before/after setup for ease of debugging" + # The builder's repr is not used in a command to avoid pointlessly + # invalidating Docker's build cache. + return self.step( + "Diagnostics", + [ + self.comment("Builder {0}".format(repr(self))), + self.run(ShellQuoted("hostname")), + self.run(ShellQuoted("cat /etc/issue || echo no /etc/issue")), + self.run(ShellQuoted("g++ --version || echo g++ not installed")), + self.run(ShellQuoted("cmake --version || echo cmake not installed")), + ], + ) + + def step(self, name, actions): + "A labeled collection of actions or other steps" + raise NotImplementedError + + def run(self, shell_cmd): + "Run this bash command" + raise NotImplementedError + + def set_env(self, key, value): + 'Set the environment "key" to value "value"' + raise NotImplementedError + + def workdir(self, dir): + "Create this directory if it does not exist, and change into it" + raise NotImplementedError + + def copy_local_repo(self, dir, dest_name): + """ + Copy the local repo at `dir` into this step's `workdir()`, analog of: + cp -r /path/to/folly folly + """ + raise NotImplementedError + + def python_deps(self): + return [ + "wheel", + "cython==0.28.6", + ] + + def debian_deps(self): + return [ + "autoconf-archive", + "bison", + "build-essential", + "cmake", + "curl", + "flex", + "git", + "gperf", + "joe", + "libboost-all-dev", + "libcap-dev", + "libdouble-conversion-dev", + "libevent-dev", + "libgflags-dev", + "libgoogle-glog-dev", + "libkrb5-dev", + "libpcre3-dev", + "libpthread-stubs0-dev", + "libnuma-dev", + "libsasl2-dev", + "libsnappy-dev", + "libsqlite3-dev", + "libssl-dev", + "libtool", + "netcat-openbsd", + "pkg-config", + "sudo", + "unzip", + "wget", + "python3-venv", + ] + + # + # Specific build helpers + # + + def install_debian_deps(self): + actions = [ + self.run( + ShellQuoted("apt-get update && apt-get install -yq {deps}").format( + deps=shell_join( + " ", (ShellQuoted(dep) for dep in self.debian_deps()) + ) + ) + ), + ] + gcc_version = self.option("gcc_version") + + # Make the selected GCC the default before building anything + actions.extend( + [ + self.run( + ShellQuoted("apt-get install -yq {c} {cpp}").format( + c=ShellQuoted("gcc-{v}").format(v=gcc_version), + cpp=ShellQuoted("g++-{v}").format(v=gcc_version), + ) + ), + self.run( + ShellQuoted( + "update-alternatives --install /usr/bin/gcc gcc {c} 40 " + "--slave /usr/bin/g++ g++ {cpp}" + ).format( + c=ShellQuoted("/usr/bin/gcc-{v}").format(v=gcc_version), + cpp=ShellQuoted("/usr/bin/g++-{v}").format(v=gcc_version), + ) + ), + self.run(ShellQuoted("update-alternatives --config gcc")), + ] + ) + + actions.extend(self.debian_ccache_setup_steps()) + + return self.step("Install packages for Debian-based OS", actions) + + def create_python_venv(self): + actions = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + actions.append( + self.run( + ShellQuoted("python3 -m venv {p}").format( + p=path_join(self.option("prefix"), "venv") + ) + ) + ) + return actions + + def python_venv(self): + actions = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + actions.append( + ShellQuoted("source {p}").format( + p=path_join(self.option("prefix"), "venv", "bin", "activate") + ) + ) + + actions.append( + self.run( + ShellQuoted("python3 -m pip install {deps}").format( + deps=shell_join( + " ", (ShellQuoted(dep) for dep in self.python_deps()) + ) + ) + ) + ) + return actions + + def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True): + choices = set(["stable", "beta", "nightly"]) + + assert toolchain in choices, ( + "while enabling rust toolchain: {} is not in {}" + ).format(toolchain, choices) + + rust_toolchain_opt = (toolchain, is_bootstrap) + prev_opt = self.option("rust_toolchain", rust_toolchain_opt) + assert prev_opt == rust_toolchain_opt, ( + "while enabling rust toolchain: previous toolchain already set to" + " {}, but trying to set it to {} now" + ).format(prev_opt, rust_toolchain_opt) + + self.add_option("rust_toolchain", rust_toolchain_opt) + + def rust_toolchain(self): + actions = [] + if self.option("rust_toolchain", False): + (toolchain, is_bootstrap) = self.option("rust_toolchain") + rust_dir = path_join(self.option("prefix"), "rust") + actions = [ + self.set_env("CARGO_HOME", rust_dir), + self.set_env("RUSTUP_HOME", rust_dir), + self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"), + self.run( + ShellQuoted( + "curl -sSf https://build.travis-ci.com/files/rustup-init.sh" + " | sh -s --" + " --default-toolchain={r} " + " --profile=minimal" + " --no-modify-path" + " -y" + ).format(p=rust_dir, r=toolchain) + ), + self.set_env( + "PATH", + ShellQuoted("{p}:$PATH").format(p=path_join(rust_dir, "bin")), + ), + self.run(ShellQuoted("rustup update")), + self.run(ShellQuoted("rustc --version")), + self.run(ShellQuoted("rustup --version")), + self.run(ShellQuoted("cargo --version")), + ] + return actions + + def debian_ccache_setup_steps(self): + return [] # It's ok to ship a renderer without ccache support. + + def github_project_workdir(self, project, path): + # Only check out a non-default branch if requested. This especially + # makes sense when building from a local repo. + git_hash = self.option( + "{0}:git_hash".format(project), + # Any repo that has a hash in deps/github_hashes defaults to + # that, with the goal of making builds maximally consistent. + self._github_hashes.get(project, ""), + ) + maybe_change_branch = ( + [ + self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)), + ] + if git_hash + else [] + ) + + local_repo_dir = self.option("{0}:local_repo_dir".format(project), "") + return self.step( + "Check out {0}, workdir {1}".format(project, path), + [ + self.workdir(self._github_dir), + self.run( + ShellQuoted("git clone {opts} https://github.com/{p}").format( + p=project, + opts=ShellQuoted( + self.option("{}:git_clone_opts".format(project), "") + ), + ) + ) + if not local_repo_dir + else self.copy_local_repo(local_repo_dir, os.path.basename(project)), + self.workdir( + path_join(self._github_dir, os.path.basename(project), path), + ), + ] + + maybe_change_branch, + ) + + def fb_github_project_workdir(self, project_and_path, github_org="facebook"): + "This helper lets Facebook-internal CI special-cases FB projects" + project, path = project_and_path.split("/", 1) + return self.github_project_workdir(github_org + "/" + project, path) + + def _make_vars(self, make_vars): + return shell_join( + " ", + ( + ShellQuoted("{k}={v}").format(k=k, v=v) + for k, v in ({} if make_vars is None else make_vars).items() + ), + ) + + def parallel_make(self, make_vars=None): + return self.run( + ShellQuoted("make -j {n} VERBOSE=1 {vars}").format( + n=self.option("make_parallelism"), + vars=self._make_vars(make_vars), + ) + ) + + def make_and_install(self, make_vars=None): + return [ + self.parallel_make(make_vars), + self.run( + ShellQuoted("make install VERBOSE=1 {vars}").format( + vars=self._make_vars(make_vars), + ) + ), + ] + + def configure(self, name=None): + autoconf_options = {} + if name is not None: + autoconf_options.update( + self.option("{0}:autoconf_options".format(name), {}) + ) + return [ + self.run( + ShellQuoted( + 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' + 'CFLAGS="$CFLAGS -I"{p}"/include" ' + 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' + "PY_PREFIX={p} " + "./configure --prefix={p} {args}" + ).format( + p=self.option("prefix"), + args=shell_join( + " ", + ( + ShellQuoted("{k}={v}").format(k=k, v=v) + for k, v in autoconf_options.items() + ), + ), + ) + ), + ] + + def autoconf_install(self, name): + return self.step( + "Build and install {0}".format(name), + [ + self.run(ShellQuoted("autoreconf -ivf")), + ] + + self.configure() + + self.make_and_install(), + ) + + def cmake_configure(self, name, cmake_path=".."): + cmake_defines = { + "BUILD_SHARED_LIBS": "ON", + "CMAKE_INSTALL_PREFIX": self.option("prefix"), + } + + # Hacks to add thriftpy3 support + if "BUILD_THRIFT_PY3" in os.environ and "folly" in name: + cmake_defines["PYTHON_EXTENSIONS"] = "True" + + if "BUILD_THRIFT_PY3" in os.environ and "fbthrift" in name: + cmake_defines["thriftpy3"] = "ON" + + cmake_defines.update(self.option("{0}:cmake_defines".format(name), {})) + return [ + self.run( + ShellQuoted( + 'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" ' + 'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" ' + "cmake {args} {cmake_path}" + ).format( + p=self.option("prefix"), + args=shell_join( + " ", + ( + ShellQuoted("-D{k}={v}").format(k=k, v=v) + for k, v in cmake_defines.items() + ), + ), + cmake_path=cmake_path, + ) + ), + ] + + def cmake_install(self, name, cmake_path=".."): + return self.step( + "Build and install {0}".format(name), + self.cmake_configure(name, cmake_path) + self.make_and_install(), + ) + + def cargo_build(self, name): + return self.step( + "Build {0}".format(name), + [ + self.run( + ShellQuoted("cargo build -j {n}").format( + n=self.option("make_parallelism") + ) + ) + ], + ) + + def fb_github_autoconf_install(self, project_and_path, github_org="facebook"): + return [ + self.fb_github_project_workdir(project_and_path, github_org), + self.autoconf_install(project_and_path), + ] + + def fb_github_cmake_install( + self, project_and_path, cmake_path="..", github_org="facebook" + ): + return [ + self.fb_github_project_workdir(project_and_path, github_org), + self.cmake_install(project_and_path, cmake_path), + ] + + def fb_github_cargo_build(self, project_and_path, github_org="facebook"): + return [ + self.fb_github_project_workdir(project_and_path, github_org), + self.cargo_build(project_and_path), + ] diff --git a/build/fbcode_builder/fbcode_builder_config.py b/build/fbcode_builder/fbcode_builder_config.py new file mode 100644 index 000000000000..5ba6e607a920 --- /dev/null +++ b/build/fbcode_builder/fbcode_builder_config.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +"Demo config, so that `make_docker_context.py --help` works in this directory." + +config = { + "fbcode_builder_spec": lambda _builder: { + "depends_on": [], + "steps": [], + }, + "github_project": "demo/project", +} diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py new file mode 100755 index 000000000000..1b539735f14f --- /dev/null +++ b/build/fbcode_builder/getdeps.py @@ -0,0 +1,1071 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile + +# We don't import cache.create_cache directly as the facebook +# specific import below may monkey patch it, and we want to +# observe the patched version of this function! +import getdeps.cache as cache_module +from getdeps.buildopts import setup_build_options +from getdeps.dyndeps import create_dyn_dep_munger +from getdeps.errors import TransientFailure +from getdeps.fetcher import ( + SystemPackageFetcher, + file_name_is_cmake_file, + list_files_under_dir_newer_than_timestamp, +) +from getdeps.load import ManifestLoader +from getdeps.manifest import ManifestParser +from getdeps.platform import HostType +from getdeps.runcmd import run_cmd +from getdeps.subcmd import SubCmd, add_subcommands, cmd + + +try: + import getdeps.facebook # noqa: F401 +except ImportError: + # we don't ship the facebook specific subdir, + # so allow that to fail silently + pass + + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps")) + + +class UsageError(Exception): + pass + + +@cmd("validate-manifest", "parse a manifest and validate that it is correct") +class ValidateManifest(SubCmd): + def run(self, args): + try: + ManifestParser(file_name=args.file_name) + print("OK", file=sys.stderr) + return 0 + except Exception as exc: + print("ERROR: %s" % str(exc), file=sys.stderr) + return 1 + + def setup_parser(self, parser): + parser.add_argument("file_name", help="path to the manifest file") + + +@cmd("show-host-type", "outputs the host type tuple for the host machine") +class ShowHostType(SubCmd): + def run(self, args): + host = HostType() + print("%s" % host.as_tuple_string()) + return 0 + + +class ProjectCmdBase(SubCmd): + def run(self, args): + opts = setup_build_options(args) + + if args.current_project is not None: + opts.repo_project = args.current_project + if args.project is None: + if opts.repo_project is None: + raise UsageError( + "no project name specified, and no .projectid file found" + ) + if opts.repo_project == "fbsource": + # The fbsource repository is a little special. There is no project + # manifest file for it. A specific project must always be explicitly + # specified when building from fbsource. + raise UsageError( + "no project name specified (required when building in fbsource)" + ) + args.project = opts.repo_project + + ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal) + if args.test_dependencies: + ctx_gen.set_value_for_all_projects("test", "on") + if args.enable_tests: + ctx_gen.set_value_for_project(args.project, "test", "on") + else: + ctx_gen.set_value_for_project(args.project, "test", "off") + + loader = ManifestLoader(opts, ctx_gen) + self.process_project_dir_arguments(args, loader) + + manifest = loader.load_manifest(args.project) + + self.run_project_cmd(args, loader, manifest) + + def process_project_dir_arguments(self, args, loader): + def parse_project_arg(arg, arg_type): + parts = arg.split(":") + if len(parts) == 2: + project, path = parts + elif len(parts) == 1: + project = args.project + path = parts[0] + # On Windows path contains colon, e.g. C:\open + elif os.name == "nt" and len(parts) == 3: + project = parts[0] + path = parts[1] + ":" + parts[2] + else: + raise UsageError( + "invalid %s argument; too many ':' characters: %s" % (arg_type, arg) + ) + + return project, os.path.abspath(path) + + # If we are currently running from a project repository, + # use the current repository for the project sources. + build_opts = loader.build_opts + if build_opts.repo_project is not None and build_opts.repo_root is not None: + loader.set_project_src_dir(build_opts.repo_project, build_opts.repo_root) + + for arg in args.src_dir: + project, path = parse_project_arg(arg, "--src-dir") + loader.set_project_src_dir(project, path) + + for arg in args.build_dir: + project, path = parse_project_arg(arg, "--build-dir") + loader.set_project_build_dir(project, path) + + for arg in args.install_dir: + project, path = parse_project_arg(arg, "--install-dir") + loader.set_project_install_dir(project, path) + + for arg in args.project_install_prefix: + project, path = parse_project_arg(arg, "--install-prefix") + loader.set_project_install_prefix(project, path) + + def setup_parser(self, parser): + parser.add_argument( + "project", + nargs="?", + help=( + "name of the project or path to a manifest " + "file describing the project" + ), + ) + parser.add_argument( + "--no-tests", + action="store_false", + dest="enable_tests", + default=True, + help="Disable building tests for this project.", + ) + parser.add_argument( + "--test-dependencies", + action="store_true", + help="Enable building tests for dependencies as well.", + ) + parser.add_argument( + "--current-project", + help="Specify the name of the fbcode_builder manifest file for the " + "current repository. If not specified, the code will attempt to find " + "this in a .projectid file in the repository root.", + ) + parser.add_argument( + "--src-dir", + default=[], + action="append", + help="Specify a local directory to use for the project source, " + "rather than fetching it.", + ) + parser.add_argument( + "--build-dir", + default=[], + action="append", + help="Explicitly specify the build directory to use for the " + "project, instead of the default location in the scratch path. " + "This only affects the project specified, and not its dependencies.", + ) + parser.add_argument( + "--install-dir", + default=[], + action="append", + help="Explicitly specify the install directory to use for the " + "project, instead of the default location in the scratch path. " + "This only affects the project specified, and not its dependencies.", + ) + parser.add_argument( + "--project-install-prefix", + default=[], + action="append", + help="Specify the final deployment installation path for a project", + ) + + self.setup_project_cmd_parser(parser) + + def setup_project_cmd_parser(self, parser): + pass + + +class CachedProject(object): + """A helper that allows calling the cache logic for a project + from both the build and the fetch code""" + + def __init__(self, cache, loader, m): + self.m = m + self.inst_dir = loader.get_project_install_dir(m) + self.project_hash = loader.get_project_hash(m) + self.ctx = loader.ctx_gen.get_context(m.name) + self.loader = loader + self.cache = cache + + self.cache_file_name = "-".join( + ( + m.name, + self.ctx.get("os"), + self.ctx.get("distro") or "none", + self.ctx.get("distro_vers") or "none", + self.project_hash, + "buildcache.tgz", + ) + ) + + def is_cacheable(self): + """We only cache third party projects""" + return self.cache and self.m.shipit_project is None + + def was_cached(self): + cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build") + return os.path.exists(cached_marker) + + def download(self): + if self.is_cacheable() and not os.path.exists(self.inst_dir): + print("check cache for %s" % self.cache_file_name) + dl_dir = os.path.join(self.loader.build_opts.scratch_dir, "downloads") + if not os.path.exists(dl_dir): + os.makedirs(dl_dir) + try: + target_file_name = os.path.join(dl_dir, self.cache_file_name) + if self.cache.download_to_file(self.cache_file_name, target_file_name): + tf = tarfile.open(target_file_name, "r") + print( + "Extracting %s -> %s..." % (self.cache_file_name, self.inst_dir) + ) + tf.extractall(self.inst_dir) + + cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build") + with open(cached_marker, "w") as f: + f.write("\n") + + return True + except Exception as exc: + print("%s" % str(exc)) + + return False + + def upload(self): + if self.is_cacheable(): + # We can prepare an archive and stick it in LFS + tempdir = tempfile.mkdtemp() + tarfilename = os.path.join(tempdir, self.cache_file_name) + print("Archiving for cache: %s..." % tarfilename) + tf = tarfile.open(tarfilename, "w:gz") + tf.add(self.inst_dir, arcname=".") + tf.close() + try: + self.cache.upload_from_file(self.cache_file_name, tarfilename) + except Exception as exc: + print( + "Failed to upload to cache (%s), continue anyway" % str(exc), + file=sys.stderr, + ) + shutil.rmtree(tempdir) + + +@cmd("fetch", "fetch the code for a given project") +class FetchCmd(ProjectCmdBase): + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--recursive", + help="fetch the transitive deps also", + action="store_true", + default=False, + ) + parser.add_argument( + "--host-type", + help=( + "When recursively fetching, fetch deps for " + "this host type rather than the current system" + ), + ) + + def run_project_cmd(self, args, loader, manifest): + if args.recursive: + projects = loader.manifests_in_dependency_order() + else: + projects = [manifest] + + cache = cache_module.create_cache() + for m in projects: + cached_project = CachedProject(cache, loader, m) + if cached_project.download(): + continue + + inst_dir = loader.get_project_install_dir(m) + built_marker = os.path.join(inst_dir, ".built-by-getdeps") + if os.path.exists(built_marker): + with open(built_marker, "r") as f: + built_hash = f.read().strip() + + project_hash = loader.get_project_hash(m) + if built_hash == project_hash: + continue + + # We need to fetch the sources + fetcher = loader.create_fetcher(m) + fetcher.update() + + +@cmd("install-system-deps", "Install system packages to satisfy the deps for a project") +class InstallSysDepsCmd(ProjectCmdBase): + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--recursive", + help="install the transitive deps also", + action="store_true", + default=False, + ) + + def run_project_cmd(self, args, loader, manifest): + if args.recursive: + projects = loader.manifests_in_dependency_order() + else: + projects = [manifest] + + cache = cache_module.create_cache() + all_packages = {} + for m in projects: + ctx = loader.ctx_gen.get_context(m.name) + packages = m.get_required_system_packages(ctx) + for k, v in packages.items(): + merged = all_packages.get(k, []) + merged += v + all_packages[k] = merged + + manager = loader.build_opts.host_type.get_package_manager() + if manager == "rpm": + packages = sorted(list(set(all_packages["rpm"]))) + if packages: + run_cmd(["dnf", "install", "-y"] + packages) + elif manager == "deb": + packages = sorted(list(set(all_packages["deb"]))) + if packages: + run_cmd(["apt", "install", "-y"] + packages) + else: + print("I don't know how to install any packages on this system") + + +@cmd("list-deps", "lists the transitive deps for a given project") +class ListDepsCmd(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + for m in loader.manifests_in_dependency_order(): + print(m.name) + return 0 + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--host-type", + help=( + "Produce the list for the specified host type, " + "rather than that of the current system" + ), + ) + + +def clean_dirs(opts): + for d in ["build", "installed", "extracted", "shipit"]: + d = os.path.join(opts.scratch_dir, d) + print("Cleaning %s..." % d) + if os.path.exists(d): + shutil.rmtree(d) + + +@cmd("clean", "clean up the scratch dir") +class CleanCmd(SubCmd): + def run(self, args): + opts = setup_build_options(args) + clean_dirs(opts) + + +@cmd("show-build-dir", "print the build dir for a given project") +class ShowBuildDirCmd(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + if args.recursive: + manifests = loader.manifests_in_dependency_order() + else: + manifests = [manifest] + + for m in manifests: + inst_dir = loader.get_project_build_dir(m) + print(inst_dir) + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--recursive", + help="print the transitive deps also", + action="store_true", + default=False, + ) + + +@cmd("show-inst-dir", "print the installation dir for a given project") +class ShowInstDirCmd(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + if args.recursive: + manifests = loader.manifests_in_dependency_order() + else: + manifests = [manifest] + + for m in manifests: + inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) + print(inst_dir) + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--recursive", + help="print the transitive deps also", + action="store_true", + default=False, + ) + + +@cmd("show-source-dir", "print the source dir for a given project") +class ShowSourceDirCmd(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + if args.recursive: + manifests = loader.manifests_in_dependency_order() + else: + manifests = [manifest] + + for m in manifests: + fetcher = loader.create_fetcher(m) + print(fetcher.get_src_dir()) + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--recursive", + help="print the transitive deps also", + action="store_true", + default=False, + ) + + +@cmd("build", "build a given project") +class BuildCmd(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + if args.clean: + clean_dirs(loader.build_opts) + + print("Building on %s" % loader.ctx_gen.get_context(args.project)) + projects = loader.manifests_in_dependency_order() + + cache = cache_module.create_cache() if args.use_build_cache else None + + # Accumulate the install directories so that the build steps + # can find their dep installation + install_dirs = [] + + for m in projects: + fetcher = loader.create_fetcher(m) + + if isinstance(fetcher, SystemPackageFetcher): + # We are guaranteed that if the fetcher is set to + # SystemPackageFetcher then this item is completely + # satisfied by the appropriate system packages + continue + + if args.clean: + fetcher.clean() + + build_dir = loader.get_project_build_dir(m) + inst_dir = loader.get_project_install_dir(m) + + if ( + m == manifest + and not args.only_deps + or m != manifest + and not args.no_deps + ): + print("Assessing %s..." % m.name) + project_hash = loader.get_project_hash(m) + ctx = loader.ctx_gen.get_context(m.name) + built_marker = os.path.join(inst_dir, ".built-by-getdeps") + + cached_project = CachedProject(cache, loader, m) + + reconfigure, sources_changed = self.compute_source_change_status( + cached_project, fetcher, m, built_marker, project_hash + ) + + if os.path.exists(built_marker) and not cached_project.was_cached(): + # We've previously built this. We may need to reconfigure if + # our deps have changed, so let's check them. + dep_reconfigure, dep_build = self.compute_dep_change_status( + m, built_marker, loader + ) + if dep_reconfigure: + reconfigure = True + if dep_build: + sources_changed = True + + extra_cmake_defines = ( + json.loads(args.extra_cmake_defines) + if args.extra_cmake_defines + else {} + ) + + if sources_changed or reconfigure or not os.path.exists(built_marker): + if os.path.exists(built_marker): + os.unlink(built_marker) + src_dir = fetcher.get_src_dir() + builder = m.create_builder( + loader.build_opts, + src_dir, + build_dir, + inst_dir, + ctx, + loader, + final_install_prefix=loader.get_project_install_prefix(m), + extra_cmake_defines=extra_cmake_defines, + ) + builder.build(install_dirs, reconfigure=reconfigure) + + with open(built_marker, "w") as f: + f.write(project_hash) + + # Only populate the cache from continuous build runs + if args.schedule_type == "continuous": + cached_project.upload() + + install_dirs.append(inst_dir) + + def compute_dep_change_status(self, m, built_marker, loader): + reconfigure = False + sources_changed = False + st = os.lstat(built_marker) + + ctx = loader.ctx_gen.get_context(m.name) + dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) + for dep in dep_list: + if reconfigure and sources_changed: + break + + dep_manifest = loader.load_manifest(dep) + dep_root = loader.get_project_install_dir(dep_manifest) + for dep_file in list_files_under_dir_newer_than_timestamp( + dep_root, st.st_mtime + ): + if os.path.basename(dep_file) == ".built-by-getdeps": + continue + if file_name_is_cmake_file(dep_file): + if not reconfigure: + reconfigure = True + print( + f"Will reconfigure cmake because {dep_file} is newer than {built_marker}" + ) + else: + if not sources_changed: + sources_changed = True + print( + f"Will run build because {dep_file} is newer than {built_marker}" + ) + + if reconfigure and sources_changed: + break + + return reconfigure, sources_changed + + def compute_source_change_status( + self, cached_project, fetcher, m, built_marker, project_hash + ): + reconfigure = False + sources_changed = False + if not cached_project.download(): + check_fetcher = True + if os.path.exists(built_marker): + check_fetcher = False + with open(built_marker, "r") as f: + built_hash = f.read().strip() + if built_hash == project_hash: + if cached_project.is_cacheable(): + # We can blindly trust the build status + reconfigure = False + sources_changed = False + else: + # Otherwise, we may have changed the source, so let's + # check in with the fetcher layer + check_fetcher = True + else: + # Some kind of inconsistency with a prior build, + # let's run it again to be sure + os.unlink(built_marker) + reconfigure = True + sources_changed = True + # While we don't need to consult the fetcher for the + # status in this case, we may still need to have eg: shipit + # run in order to have a correct source tree. + fetcher.update() + + if check_fetcher: + change_status = fetcher.update() + reconfigure = change_status.build_changed() + sources_changed = change_status.sources_changed() + + return reconfigure, sources_changed + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--clean", + action="store_true", + default=False, + help=( + "Clean up the build and installation area prior to building, " + "causing the projects to be built from scratch" + ), + ) + parser.add_argument( + "--no-deps", + action="store_true", + default=False, + help=( + "Only build the named project, not its deps. " + "This is most useful after you've built all of the deps, " + "and helps to avoid waiting for relatively " + "slow up-to-date-ness checks" + ), + ) + parser.add_argument( + "--only-deps", + action="store_true", + default=False, + help=( + "Only build the named project's deps. " + "This is most useful when you want to separate out building " + "of all of the deps and your project" + ), + ) + parser.add_argument( + "--no-build-cache", + action="store_false", + default=True, + dest="use_build_cache", + help="Do not attempt to use the build cache.", + ) + parser.add_argument( + "--schedule-type", help="Indicates how the build was activated" + ) + parser.add_argument( + "--extra-cmake-defines", + help=( + "Input json map that contains extra cmake defines to be used " + "when compiling the current project and all its deps. " + 'e.g: \'{"CMAKE_CXX_FLAGS": "--bla"}\'' + ), + ) + + +@cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes") +class FixupDeps(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + projects = loader.manifests_in_dependency_order() + + # Accumulate the install directories so that the build steps + # can find their dep installation + install_dirs = [] + + for m in projects: + inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) + install_dirs.append(inst_dir) + + if m == manifest: + dep_munger = create_dyn_dep_munger( + loader.build_opts, install_dirs, args.strip + ) + dep_munger.process_deps(args.destdir, args.final_install_prefix) + + def setup_project_cmd_parser(self, parser): + parser.add_argument("destdir", help="Where to copy the fixed up executables") + parser.add_argument( + "--final-install-prefix", help="specify the final installation prefix" + ) + parser.add_argument( + "--strip", + action="store_true", + default=False, + help="Strip debug info while processing executables", + ) + + +@cmd("test", "test a given project") +class TestCmd(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + projects = loader.manifests_in_dependency_order() + + # Accumulate the install directories so that the test steps + # can find their dep installation + install_dirs = [] + + for m in projects: + inst_dir = loader.get_project_install_dir(m) + + if m == manifest or args.test_dependencies: + built_marker = os.path.join(inst_dir, ".built-by-getdeps") + if not os.path.exists(built_marker): + print("project %s has not been built" % m.name) + # TODO: we could just go ahead and build it here, but I + # want to tackle that as part of adding build-for-test + # support. + return 1 + fetcher = loader.create_fetcher(m) + src_dir = fetcher.get_src_dir() + ctx = loader.ctx_gen.get_context(m.name) + build_dir = loader.get_project_build_dir(m) + builder = m.create_builder( + loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader + ) + + builder.run_tests( + install_dirs, + schedule_type=args.schedule_type, + owner=args.test_owner, + test_filter=args.filter, + retry=args.retry, + no_testpilot=args.no_testpilot, + ) + + install_dirs.append(inst_dir) + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--schedule-type", help="Indicates how the build was activated" + ) + parser.add_argument("--test-owner", help="Owner for testpilot") + parser.add_argument("--filter", help="Only run the tests matching the regex") + parser.add_argument( + "--retry", + type=int, + default=3, + help="Number of immediate retries for failed tests " + "(noop in continuous and testwarden runs)", + ) + parser.add_argument( + "--no-testpilot", + help="Do not use Test Pilot even when available", + action="store_true", + ) + + +@cmd("generate-github-actions", "generate a GitHub actions configuration") +class GenerateGitHubActionsCmd(ProjectCmdBase): + RUN_ON_ALL = """ [push, pull_request]""" + RUN_ON_DEFAULT = """ + push: + branches: + - master + pull_request: + branches: + - master""" + + def run_project_cmd(self, args, loader, manifest): + platforms = [ + HostType("linux", "ubuntu", "18"), + HostType("darwin", None, None), + HostType("windows", None, None), + ] + + for p in platforms: + self.write_job_for_platform(p, args) + + # TODO: Break up complex function + def write_job_for_platform(self, platform, args): # noqa: C901 + build_opts = setup_build_options(args, platform) + ctx_gen = build_opts.get_context_generator(facebook_internal=False) + loader = ManifestLoader(build_opts, ctx_gen) + manifest = loader.load_manifest(args.project) + manifest_ctx = loader.ctx_gen.get_context(manifest.name) + run_on = self.RUN_ON_ALL if args.run_on_all_branches else self.RUN_ON_DEFAULT + + # Some projects don't do anything "useful" as a leaf project, only + # as a dep for a leaf project. Check for those here; we don't want + # to waste the effort scheduling them on CI. + # We do this by looking at the builder type in the manifest file + # rather than creating a builder and checking its type because we + # don't know enough to create the full builder instance here. + if manifest.get("build", "builder", ctx=manifest_ctx) == "nop": + return None + + # We want to be sure that we're running things with python 3 + # but python versioning is honestly a bit of a frustrating mess. + # `python` may be version 2 or version 3 depending on the system. + # python3 may not be a thing at all! + # Assume an optimistic default + py3 = "python3" + + if build_opts.is_linux(): + job_name = "linux" + runs_on = f"ubuntu-{args.ubuntu_version}" + elif build_opts.is_windows(): + # We're targeting the windows-2016 image because it has + # Visual Studio 2017 installed, and at the time of writing, + # the version of boost in the manifests (1.69) is not + # buildable with Visual Studio 2019 + job_name = "windows" + runs_on = "windows-2016" + # The windows runners are python 3 by default; python2.exe + # is available if needed. + py3 = "python" + else: + job_name = "mac" + runs_on = "macOS-latest" + + os.makedirs(args.output_dir, exist_ok=True) + output_file = os.path.join(args.output_dir, f"getdeps_{job_name}.yml") + with open(output_file, "w") as out: + # Deliberate line break here because the @ and the generated + # symbols are meaningful to our internal tooling when they + # appear in a single token + out.write("# This file was @") + out.write("generated by getdeps.py\n") + out.write( + f""" +name: {job_name} + +on:{run_on} + +jobs: +""" + ) + + getdeps = f"{py3} build/fbcode_builder/getdeps.py" + + out.write(" build:\n") + out.write(" runs-on: %s\n" % runs_on) + out.write(" steps:\n") + out.write(" - uses: actions/checkout@v1\n") + + if build_opts.is_windows(): + # cmake relies on BOOST_ROOT but GH deliberately don't set it in order + # to avoid versioning issues: + # https://github.com/actions/virtual-environments/issues/319 + # Instead, set the version we think we need; this is effectively + # coupled with the boost manifest + # This is the unusual syntax for setting an env var for the rest of + # the steps in a workflow: + # https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ + out.write(" - name: Export boost environment\n") + out.write( + ' run: "echo BOOST_ROOT=%BOOST_ROOT_1_69_0% >> %GITHUB_ENV%"\n' + ) + out.write(" shell: cmd\n") + + # The git installation may not like long filenames, so tell it + # that we want it to use them! + out.write(" - name: Fix Git config\n") + out.write(" run: git config --system core.longpaths true\n") + + projects = loader.manifests_in_dependency_order() + + for m in projects: + if m != manifest: + out.write(" - name: Fetch %s\n" % m.name) + out.write(f" run: {getdeps} fetch --no-tests {m.name}\n") + + for m in projects: + if m != manifest: + out.write(" - name: Build %s\n" % m.name) + out.write(f" run: {getdeps} build --no-tests {m.name}\n") + + out.write(" - name: Build %s\n" % manifest.name) + + project_prefix = "" + if not build_opts.is_windows(): + project_prefix = ( + " --project-install-prefix %s:/usr/local" % manifest.name + ) + + out.write( + f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n" + ) + + out.write(" - name: Copy artifacts\n") + if build_opts.is_linux(): + # Strip debug info from the binaries, but only on linux. + # While the `strip` utility is also available on macOS, + # attempting to strip there results in an error. + # The `strip` utility is not available on Windows. + strip = " --strip" + else: + strip = "" + + out.write( + f" run: {getdeps} fixup-dyn-deps{strip} " + f"--src-dir=. {manifest.name} _artifacts/{job_name} {project_prefix} " + f"--final-install-prefix /usr/local\n" + ) + + out.write(" - uses: actions/upload-artifact@master\n") + out.write(" with:\n") + out.write(" name: %s\n" % manifest.name) + out.write(" path: _artifacts\n") + + out.write(" - name: Test %s\n" % manifest.name) + out.write( + f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n" + ) + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--disallow-system-packages", + help="Disallow satisfying third party deps from installed system packages", + action="store_true", + default=False, + ) + parser.add_argument( + "--output-dir", help="The directory that will contain the yml files" + ) + parser.add_argument( + "--run-on-all-branches", + action="store_true", + help="Allow CI to fire on all branches - Handy for testing", + ) + parser.add_argument( + "--ubuntu-version", default="18.04", help="Version of Ubuntu to use" + ) + + +def get_arg_var_name(args): + for arg in args: + if arg.startswith("--"): + return arg[2:].replace("-", "_") + + raise Exception("unable to determine argument variable name from %r" % (args,)) + + +def parse_args(): + # We want to allow common arguments to be specified either before or after + # the subcommand name. In order to do this we add them to the main parser + # and to subcommand parsers. In order for this to work, we need to tell + # argparse that the default value is SUPPRESS, so that the default values + # from the subparser arguments won't override values set by the user from + # the main parser. We maintain our own list of desired defaults in the + # common_defaults dictionary, and manually set those if the argument wasn't + # present at all. + common_args = argparse.ArgumentParser(add_help=False) + common_defaults = {} + + def add_common_arg(*args, **kwargs): + var_name = get_arg_var_name(args) + default_value = kwargs.pop("default", None) + common_defaults[var_name] = default_value + kwargs["default"] = argparse.SUPPRESS + common_args.add_argument(*args, **kwargs) + + add_common_arg("--scratch-path", help="Where to maintain checkouts and build dirs") + add_common_arg( + "--vcvars-path", default=None, help="Path to the vcvarsall.bat on Windows." + ) + add_common_arg( + "--install-prefix", + help=( + "Where the final build products will be installed " + "(default is [scratch-path]/installed)" + ), + ) + add_common_arg( + "--num-jobs", + type=int, + help=( + "Number of concurrent jobs to use while building. " + "(default=number of cpu cores)" + ), + ) + add_common_arg( + "--use-shipit", + help="use the real ShipIt instead of the simple shipit transformer", + action="store_true", + default=False, + ) + add_common_arg( + "--facebook-internal", + help="Setup the build context as an FB internal build", + action="store_true", + default=None, + ) + add_common_arg( + "--no-facebook-internal", + help="Perform a non-FB internal build, even when in an fbsource repository", + action="store_false", + dest="facebook_internal", + ) + add_common_arg( + "--allow-system-packages", + help="Allow satisfying third party deps from installed system packages", + action="store_true", + default=False, + ) + add_common_arg( + "--lfs-path", + help="Provide a parent directory for lfs when fbsource is unavailable", + default=None, + ) + + ap = argparse.ArgumentParser( + description="Get and build dependencies and projects", parents=[common_args] + ) + sub = ap.add_subparsers( + # metavar suppresses the long and ugly default list of subcommands on a + # single line. We still render the nicer list below where we would + # have shown the nasty one. + metavar="", + title="Available commands", + help="", + ) + + add_subcommands(sub, common_args) + + args = ap.parse_args() + for var_name, default_value in common_defaults.items(): + if not hasattr(args, var_name): + setattr(args, var_name, default_value) + + return ap, args + + +def main(): + ap, args = parse_args() + if getattr(args, "func", None) is None: + ap.print_help() + return 0 + try: + return args.func(args) + except UsageError as exc: + ap.error(str(exc)) + return 1 + except TransientFailure as exc: + print("TransientFailure: %s" % str(exc)) + # This return code is treated as a retryable transient infrastructure + # error by Facebook's internal CI, rather than eg: a build or code + # related error that needs to be fixed before progress can be made. + return 128 + except subprocess.CalledProcessError as exc: + print("%s" % str(exc), file=sys.stderr) + print("!! Failed", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build/fbcode_builder/getdeps/__init__.py b/build/fbcode_builder/getdeps/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py new file mode 100644 index 000000000000..4e523c2dca4f --- /dev/null +++ b/build/fbcode_builder/getdeps/builder.py @@ -0,0 +1,1400 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import os +import shutil +import stat +import subprocess +import sys + +from .dyndeps import create_dyn_dep_munger +from .envfuncs import Env, add_path_entry, path_search +from .fetcher import copy_if_different +from .runcmd import run_cmd + + +class BuilderBase(object): + def __init__( + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + env=None, + final_install_prefix=None, + ): + self.env = Env() + if env: + self.env.update(env) + + subdir = manifest.get("build", "subdir", ctx=ctx) + if subdir: + src_dir = os.path.join(src_dir, subdir) + + self.ctx = ctx + self.src_dir = src_dir + self.build_dir = build_dir or src_dir + self.inst_dir = inst_dir + self.build_opts = build_opts + self.manifest = manifest + self.final_install_prefix = final_install_prefix + + def _get_cmd_prefix(self): + if self.build_opts.is_windows(): + vcvarsall = self.build_opts.get_vcvars_path() + if vcvarsall is not None: + # Since it sets rather a large number of variables we mildly abuse + # the cmd quoting rules to assemble a command that calls the script + # to prep the environment and then triggers the actual command that + # we wanted to run. + return [vcvarsall, "amd64", "&&"] + return [] + + def _run_cmd(self, cmd, cwd=None, env=None, use_cmd_prefix=True, allow_fail=False): + if env: + e = self.env.copy() + e.update(env) + env = e + else: + env = self.env + + if use_cmd_prefix: + cmd_prefix = self._get_cmd_prefix() + if cmd_prefix: + cmd = cmd_prefix + cmd + + log_file = os.path.join(self.build_dir, "getdeps_build.log") + return run_cmd( + cmd=cmd, + env=env, + cwd=cwd or self.build_dir, + log_file=log_file, + allow_fail=allow_fail, + ) + + def build(self, install_dirs, reconfigure): + print("Building %s..." % self.manifest.name) + + if self.build_dir is not None: + if not os.path.isdir(self.build_dir): + os.makedirs(self.build_dir) + reconfigure = True + + self._build(install_dirs=install_dirs, reconfigure=reconfigure) + + # On Windows, emit a wrapper script that can be used to run build artifacts + # directly from the build directory, without installing them. On Windows $PATH + # needs to be updated to include all of the directories containing the runtime + # library dependencies in order to run the binaries. + if self.build_opts.is_windows(): + script_path = self.get_dev_run_script_path() + dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs) + dep_dirs = self.get_dev_run_extra_path_dirs(install_dirs, dep_munger) + dep_munger.emit_dev_run_script(script_path, dep_dirs) + + def run_tests( + self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot + ): + """Execute any tests that we know how to run. If they fail, + raise an exception.""" + pass + + def _build(self, install_dirs, reconfigure): + """Perform the build. + install_dirs contains the list of installation directories for + the dependencies of this project. + reconfigure will be set to true if the fetcher determined + that the sources have changed in such a way that the build + system needs to regenerate its rules.""" + pass + + def _compute_env(self, install_dirs): + # CMAKE_PREFIX_PATH is only respected when passed through the + # environment, so we construct an appropriate path to pass down + return self.build_opts.compute_env_for_install_dirs( + install_dirs, env=self.env, manifest=self.manifest + ) + + def get_dev_run_script_path(self): + assert self.build_opts.is_windows() + return os.path.join(self.build_dir, "run.ps1") + + def get_dev_run_extra_path_dirs(self, install_dirs, dep_munger=None): + assert self.build_opts.is_windows() + if dep_munger is None: + dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs) + return dep_munger.compute_dependency_paths(self.build_dir) + + +class MakeBuilder(BuilderBase): + def __init__( + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + build_args, + install_args, + test_args, + ): + super(MakeBuilder, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + self.build_args = build_args or [] + self.install_args = install_args or [] + self.test_args = test_args + + def _get_prefix(self): + return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] + + def _build(self, install_dirs, reconfigure): + env = self._compute_env(install_dirs) + + # Need to ensure that PREFIX is set prior to install because + # libbpf uses it when generating its pkg-config file. + # The lowercase prefix is used by some projects. + cmd = ( + ["make", "-j%s" % self.build_opts.num_jobs] + + self.build_args + + self._get_prefix() + ) + self._run_cmd(cmd, env=env) + + install_cmd = ["make"] + self.install_args + self._get_prefix() + self._run_cmd(install_cmd, env=env) + + def run_tests( + self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot + ): + if not self.test_args: + return + + env = self._compute_env(install_dirs) + + cmd = ["make"] + self.test_args + self._get_prefix() + self._run_cmd(cmd, env=env) + + +class CMakeBootStrapBuilder(MakeBuilder): + def _build(self, install_dirs, reconfigure): + self._run_cmd(["./bootstrap", "--prefix=" + self.inst_dir]) + super(CMakeBootStrapBuilder, self)._build(install_dirs, reconfigure) + + +class AutoconfBuilder(BuilderBase): + def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args): + super(AutoconfBuilder, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + self.args = args or [] + + def _build(self, install_dirs, reconfigure): + configure_path = os.path.join(self.src_dir, "configure") + autogen_path = os.path.join(self.src_dir, "autogen.sh") + + env = self._compute_env(install_dirs) + + if not os.path.exists(configure_path): + print("%s doesn't exist, so reconfiguring" % configure_path) + # This libtoolize call is a bit gross; the issue is that + # `autoreconf` as invoked by libsodium's `autogen.sh` doesn't + # seem to realize that it should invoke libtoolize and then + # error out when the configure script references a libtool + # related symbol. + self._run_cmd(["libtoolize"], cwd=self.src_dir, env=env) + + # We generally prefer to call the `autogen.sh` script provided + # by the project on the basis that it may know more than plain + # autoreconf does. + if os.path.exists(autogen_path): + self._run_cmd(["bash", autogen_path], cwd=self.src_dir, env=env) + else: + self._run_cmd(["autoreconf", "-ivf"], cwd=self.src_dir, env=env) + configure_cmd = [configure_path, "--prefix=" + self.inst_dir] + self.args + self._run_cmd(configure_cmd, env=env) + self._run_cmd(["make", "-j%s" % self.build_opts.num_jobs], env=env) + self._run_cmd(["make", "install"], env=env) + + +class Iproute2Builder(BuilderBase): + # ./configure --prefix does not work for iproute2. + # Thus, explicitly copy sources from src_dir to build_dir, bulid, + # and then install to inst_dir using DESTDIR + # lastly, also copy include from build_dir to inst_dir + def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir): + super(Iproute2Builder, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + + def _patch(self): + # FBOSS build currently depends on an old version of iproute2 (commit + # 7ca63aef7d1b0c808da0040c6b366ef7a61f38c1). This is missing a commit + # (ae717baf15fb4d30749ada3948d9445892bac239) needed to build iproute2 + # successfully. Apply it viz.: include stdint.h + # Reference: https://fburl.com/ilx9g5xm + with open(self.build_dir + "/tc/tc_core.c", "r") as f: + data = f.read() + + with open(self.build_dir + "/tc/tc_core.c", "w") as f: + f.write("#include \n") + f.write(data) + + def _build(self, install_dirs, reconfigure): + configure_path = os.path.join(self.src_dir, "configure") + + env = self.env.copy() + self._run_cmd([configure_path], env=env) + shutil.rmtree(self.build_dir) + shutil.copytree(self.src_dir, self.build_dir) + self._patch() + self._run_cmd(["make", "-j%s" % self.build_opts.num_jobs], env=env) + install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir] + + for d in ["include", "lib"]: + if not os.path.isdir(os.path.join(self.inst_dir, d)): + shutil.copytree( + os.path.join(self.build_dir, d), os.path.join(self.inst_dir, d) + ) + + self._run_cmd(install_cmd, env=env) + + +class BistroBuilder(BuilderBase): + def _build(self, install_dirs, reconfigure): + p = os.path.join(self.src_dir, "bistro", "bistro") + env = self._compute_env(install_dirs) + env["PATH"] = env["PATH"] + ":" + os.path.join(p, "bin") + env["TEMPLATES_PATH"] = os.path.join(p, "include", "thrift", "templates") + self._run_cmd( + [ + os.path.join(".", "cmake", "run-cmake.sh"), + "Release", + "-DCMAKE_INSTALL_PREFIX=" + self.inst_dir, + ], + cwd=p, + env=env, + ) + self._run_cmd( + [ + "make", + "install", + "-j", + str(self.build_opts.num_jobs), + ], + cwd=os.path.join(p, "cmake", "Release"), + env=env, + ) + + def run_tests( + self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot + ): + env = self._compute_env(install_dirs) + build_dir = os.path.join(self.src_dir, "bistro", "bistro", "cmake", "Release") + NUM_RETRIES = 5 + for i in range(NUM_RETRIES): + cmd = ["ctest", "--output-on-failure"] + if i > 0: + cmd.append("--rerun-failed") + cmd.append(build_dir) + try: + self._run_cmd( + cmd, + cwd=build_dir, + env=env, + ) + except Exception: + print(f"Tests failed... retrying ({i+1}/{NUM_RETRIES})") + else: + return + raise Exception(f"Tests failed even after {NUM_RETRIES} retries") + + +class CMakeBuilder(BuilderBase): + MANUAL_BUILD_SCRIPT = """\ +#!{sys.executable} + +from __future__ import absolute_import, division, print_function, unicode_literals + +import argparse +import subprocess +import sys + +CMAKE = {cmake!r} +CTEST = {ctest!r} +SRC_DIR = {src_dir!r} +BUILD_DIR = {build_dir!r} +INSTALL_DIR = {install_dir!r} +CMD_PREFIX = {cmd_prefix!r} +CMAKE_ENV = {env_str} +CMAKE_DEFINE_ARGS = {define_args_str} + + +def get_jobs_argument(num_jobs_arg: int) -> str: + if num_jobs_arg > 0: + return "-j" + str(num_jobs_arg) + + import multiprocessing + num_jobs = multiprocessing.cpu_count() // 2 + return "-j" + str(num_jobs) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "cmake_args", + nargs=argparse.REMAINDER, + help='Any extra arguments after an "--" argument will be passed ' + "directly to CMake." + ) + ap.add_argument( + "--mode", + choices=["configure", "build", "install", "test"], + default="configure", + help="The mode to run: configure, build, or install. " + "Defaults to configure", + ) + ap.add_argument( + "--build", + action="store_const", + const="build", + dest="mode", + help="An alias for --mode=build", + ) + ap.add_argument( + "-j", + "--num-jobs", + action="store", + type=int, + default=0, + help="Run the build or tests with the specified number of parallel jobs", + ) + ap.add_argument( + "--install", + action="store_const", + const="install", + dest="mode", + help="An alias for --mode=install", + ) + ap.add_argument( + "--test", + action="store_const", + const="test", + dest="mode", + help="An alias for --mode=test", + ) + args = ap.parse_args() + + # Strip off a leading "--" from the additional CMake arguments + if args.cmake_args and args.cmake_args[0] == "--": + args.cmake_args = args.cmake_args[1:] + + env = CMAKE_ENV + + if args.mode == "configure": + full_cmd = CMD_PREFIX + [CMAKE, SRC_DIR] + CMAKE_DEFINE_ARGS + args.cmake_args + elif args.mode in ("build", "install"): + target = "all" if args.mode == "build" else "install" + full_cmd = CMD_PREFIX + [ + CMAKE, + "--build", + BUILD_DIR, + "--target", + target, + "--config", + "Release", + get_jobs_argument(args.num_jobs), + ] + args.cmake_args + elif args.mode == "test": + full_cmd = CMD_PREFIX + [ + {dev_run_script}CTEST, + "--output-on-failure", + get_jobs_argument(args.num_jobs), + ] + args.cmake_args + else: + ap.error("unknown invocation mode: %s" % (args.mode,)) + + cmd_str = " ".join(full_cmd) + print("Running: %r" % (cmd_str,)) + proc = subprocess.run(full_cmd, env=env, cwd=BUILD_DIR) + sys.exit(proc.returncode) + + +if __name__ == "__main__": + main() +""" + + def __init__( + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + defines, + final_install_prefix=None, + extra_cmake_defines=None, + ): + super(CMakeBuilder, self).__init__( + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + final_install_prefix=final_install_prefix, + ) + self.defines = defines or {} + if extra_cmake_defines: + self.defines.update(extra_cmake_defines) + + def _invalidate_cache(self): + for name in [ + "CMakeCache.txt", + "CMakeFiles/CMakeError.log", + "CMakeFiles/CMakeOutput.log", + ]: + name = os.path.join(self.build_dir, name) + if os.path.isdir(name): + shutil.rmtree(name) + elif os.path.exists(name): + os.unlink(name) + + def _needs_reconfigure(self): + for name in ["CMakeCache.txt", "build.ninja"]: + name = os.path.join(self.build_dir, name) + if not os.path.exists(name): + return True + return False + + def _write_build_script(self, **kwargs): + env_lines = [" {!r}: {!r},".format(k, v) for k, v in kwargs["env"].items()] + kwargs["env_str"] = "\n".join(["{"] + env_lines + ["}"]) + + if self.build_opts.is_windows(): + kwargs["dev_run_script"] = '"powershell.exe", {!r}, '.format( + self.get_dev_run_script_path() + ) + else: + kwargs["dev_run_script"] = "" + + define_arg_lines = ["["] + for arg in kwargs["define_args"]: + # Replace the CMAKE_INSTALL_PREFIX argument to use the INSTALL_DIR + # variable that we define in the MANUAL_BUILD_SCRIPT code. + if arg.startswith("-DCMAKE_INSTALL_PREFIX="): + value = " {!r}.format(INSTALL_DIR),".format( + "-DCMAKE_INSTALL_PREFIX={}" + ) + else: + value = " {!r},".format(arg) + define_arg_lines.append(value) + define_arg_lines.append("]") + kwargs["define_args_str"] = "\n".join(define_arg_lines) + + # In order to make it easier for developers to manually run builds for + # CMake-based projects, write out some build scripts that can be used to invoke + # CMake manually. + build_script_path = os.path.join(self.build_dir, "run_cmake.py") + script_contents = self.MANUAL_BUILD_SCRIPT.format(**kwargs) + with open(build_script_path, "wb") as f: + f.write(script_contents.encode()) + os.chmod(build_script_path, 0o755) + + def _compute_cmake_define_args(self, env): + defines = { + "CMAKE_INSTALL_PREFIX": self.final_install_prefix or self.inst_dir, + "BUILD_SHARED_LIBS": "OFF", + # Some of the deps (rsocket) default to UBSAN enabled if left + # unspecified. Some of the deps fail to compile in release mode + # due to warning->error promotion. RelWithDebInfo is the happy + # medium. + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + } + if "SANDCASTLE" not in os.environ: + # We sometimes see intermittent ccache related breakages on some + # of the FB internal CI hosts, so we prefer to disable ccache + # when running in that environment. + ccache = path_search(env, "ccache") + if ccache: + defines["CMAKE_CXX_COMPILER_LAUNCHER"] = ccache + else: + # rocksdb does its own probing for ccache. + # Ensure that it is disabled on sandcastle + env["CCACHE_DISABLE"] = "1" + # Some sandcastle hosts have broken ccache related dirs, and + # even though we've asked for it to be disabled ccache is + # still invoked by rocksdb's cmake. + # Redirect its config directory to somewhere that is guaranteed + # fresh to us, and that won't have any ccache data inside. + env["CCACHE_DIR"] = f"{self.build_opts.scratch_dir}/ccache" + + if "GITHUB_ACTIONS" in os.environ and self.build_opts.is_windows(): + # GitHub actions: the host has both gcc and msvc installed, and + # the default behavior of cmake is to prefer gcc. + # Instruct cmake that we want it to use cl.exe; this is important + # because Boost prefers cl.exe and the mismatch results in cmake + # with gcc not being able to find boost built with cl.exe. + defines["CMAKE_C_COMPILER"] = "cl.exe" + defines["CMAKE_CXX_COMPILER"] = "cl.exe" + + if self.build_opts.is_darwin(): + # Try to persuade cmake to set the rpath to match the lib + # dirs of the dependencies. This isn't automatic, and to + # make things more interesting, cmake uses `;` as the path + # separator, so translate the runtime path to something + # that cmake will parse + defines["CMAKE_INSTALL_RPATH"] = ";".join( + env.get("DYLD_LIBRARY_PATH", "").split(":") + ) + # Tell cmake that we want to set the rpath in the tree + # at build time. Without this the rpath is only set + # at the moment that the binaries are installed. That + # default is problematic for example when using the + # gtest integration in cmake which runs the built test + # executables during the build to discover the set of + # tests. + defines["CMAKE_BUILD_WITH_INSTALL_RPATH"] = "ON" + + defines.update(self.defines) + define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()] + + # if self.build_opts.is_windows(): + # define_args += ["-G", "Visual Studio 15 2017 Win64"] + define_args += ["-G", "Ninja"] + + return define_args + + def _build(self, install_dirs, reconfigure): + reconfigure = reconfigure or self._needs_reconfigure() + + env = self._compute_env(install_dirs) + if not self.build_opts.is_windows() and self.final_install_prefix: + env["DESTDIR"] = self.inst_dir + + # Resolve the cmake that we installed + cmake = path_search(env, "cmake") + if cmake is None: + raise Exception("Failed to find CMake") + + if reconfigure: + define_args = self._compute_cmake_define_args(env) + self._write_build_script( + cmd_prefix=self._get_cmd_prefix(), + cmake=cmake, + ctest=path_search(env, "ctest"), + env=env, + define_args=define_args, + src_dir=self.src_dir, + build_dir=self.build_dir, + install_dir=self.inst_dir, + sys=sys, + ) + + self._invalidate_cache() + self._run_cmd([cmake, self.src_dir] + define_args, env=env) + + self._run_cmd( + [ + cmake, + "--build", + self.build_dir, + "--target", + "install", + "--config", + "Release", + "-j", + str(self.build_opts.num_jobs), + ], + env=env, + ) + + def run_tests( + self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot + ): + env = self._compute_env(install_dirs) + ctest = path_search(env, "ctest") + cmake = path_search(env, "cmake") + + # On Windows, we also need to update $PATH to include the directories that + # contain runtime library dependencies. This is not needed on other platforms + # since CMake will emit RPATH properly in the binary so they can find these + # dependencies. + if self.build_opts.is_windows(): + path_entries = self.get_dev_run_extra_path_dirs(install_dirs) + path = env.get("PATH") + if path: + path_entries.insert(0, path) + env["PATH"] = ";".join(path_entries) + + # Don't use the cmd_prefix when running tests. This is vcvarsall.bat on + # Windows. vcvarsall.bat is only needed for the build, not tests. It + # unfortunately fails if invoked with a long PATH environment variable when + # running the tests. + use_cmd_prefix = False + + def get_property(test, propname, defval=None): + """extracts a named property from a cmake test info json blob. + The properties look like: + [{"name": "WORKING_DIRECTORY"}, + {"value": "something"}] + We assume that it is invalid for the same named property to be + listed more than once. + """ + props = test.get("properties", []) + for p in props: + if p.get("name", None) == propname: + return p.get("value", defval) + return defval + + def list_tests(): + output = subprocess.check_output( + [ctest, "--show-only=json-v1"], env=env, cwd=self.build_dir + ) + try: + data = json.loads(output.decode("utf-8")) + except ValueError as exc: + raise Exception( + "Failed to decode cmake test info using %s: %s. Output was: %r" + % (ctest, str(exc), output) + ) + + tests = [] + machine_suffix = self.build_opts.host_type.as_tuple_string() + for test in data["tests"]: + working_dir = get_property(test, "WORKING_DIRECTORY") + labels = [] + machine_suffix = self.build_opts.host_type.as_tuple_string() + labels.append("tpx_test_config::buildsystem=getdeps") + labels.append("tpx_test_config::platform={}".format(machine_suffix)) + + if get_property(test, "DISABLED"): + labels.append("disabled") + command = test["command"] + if working_dir: + command = [cmake, "-E", "chdir", working_dir] + command + + import os + + tests.append( + { + "type": "custom", + "target": "%s-%s-getdeps-%s" + % (self.manifest.name, test["name"], machine_suffix), + "command": command, + "labels": labels, + "env": {}, + "required_paths": [], + "contacts": [], + "cwd": os.getcwd(), + } + ) + return tests + + if schedule_type == "continuous" or schedule_type == "testwarden": + # for continuous and testwarden runs, disabling retry can give up + # better signals for flaky tests. + retry = 0 + + from sys import platform + + testpilot = path_search(env, "testpilot") + tpx = path_search(env, "tpx") + if (tpx or testpilot) and not no_testpilot: + buck_test_info = list_tests() + import os + + buck_test_info_name = os.path.join(self.build_dir, ".buck-test-info.json") + with open(buck_test_info_name, "w") as f: + json.dump(buck_test_info, f) + + env.set("http_proxy", "") + env.set("https_proxy", "") + runs = [] + from sys import platform + + if platform == "win32": + machine_suffix = self.build_opts.host_type.as_tuple_string() + testpilot_args = [ + "parexec-testinfra.exe", + "C:/tools/testpilot/sc_testpilot.par", + # Need to force the repo type otherwise testpilot on windows + # can be confused (presumably sparse profile related) + "--force-repo", + "fbcode", + "--force-repo-root", + self.build_opts.fbsource_dir, + "--buck-test-info", + buck_test_info_name, + "--retry=%d" % retry, + "-j=%s" % str(self.build_opts.num_jobs), + "--test-config", + "platform=%s" % machine_suffix, + "buildsystem=getdeps", + "--return-nonzero-on-failures", + ] + else: + testpilot_args = [ + tpx, + "--buck-test-info", + buck_test_info_name, + "--retry=%d" % retry, + "-j=%s" % str(self.build_opts.num_jobs), + "--print-long-results", + ] + + if owner: + testpilot_args += ["--contacts", owner] + + if tpx and env: + testpilot_args.append("--env") + testpilot_args.extend(f"{key}={val}" for key, val in env.items()) + + if test_filter: + testpilot_args += ["--", test_filter] + + if schedule_type == "continuous": + runs.append( + [ + "--tag-new-tests", + "--collection", + "oss-continuous", + "--purpose", + "continuous", + ] + ) + elif schedule_type == "testwarden": + # One run to assess new tests + runs.append( + [ + "--tag-new-tests", + "--collection", + "oss-new-test-stress", + "--stress-runs", + "10", + "--purpose", + "stress-run-new-test", + ] + ) + # And another for existing tests + runs.append( + [ + "--tag-new-tests", + "--collection", + "oss-existing-test-stress", + "--stress-runs", + "10", + "--purpose", + "stress-run", + ] + ) + else: + runs.append(["--collection", "oss-diff", "--purpose", "diff"]) + + for run in runs: + self._run_cmd( + testpilot_args + run, + cwd=self.build_opts.fbcode_builder_dir, + env=env, + use_cmd_prefix=use_cmd_prefix, + ) + else: + args = [ctest, "--output-on-failure", "-j", str(self.build_opts.num_jobs)] + if test_filter: + args += ["-R", test_filter] + + count = 0 + while count <= retry: + retcode = self._run_cmd( + args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True + ) + + if retcode == 0: + break + if count == 0: + # Only add this option in the second run. + args += ["--rerun-failed"] + count += 1 + if retcode != 0: + # Allow except clause in getdeps.main to catch and exit gracefully + # This allows non-testpilot runs to fail through the same logic as failed testpilot runs, which may become handy in case if post test processing is needed in the future + raise subprocess.CalledProcessError(retcode, args) + + +class NinjaBootstrap(BuilderBase): + def __init__(self, build_opts, ctx, manifest, build_dir, src_dir, inst_dir): + super(NinjaBootstrap, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + + def _build(self, install_dirs, reconfigure): + self._run_cmd([sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir) + src_ninja = os.path.join(self.src_dir, "ninja") + dest_ninja = os.path.join(self.inst_dir, "bin/ninja") + bin_dir = os.path.dirname(dest_ninja) + if not os.path.exists(bin_dir): + os.makedirs(bin_dir) + shutil.copyfile(src_ninja, dest_ninja) + shutil.copymode(src_ninja, dest_ninja) + + +class OpenSSLBuilder(BuilderBase): + def __init__(self, build_opts, ctx, manifest, build_dir, src_dir, inst_dir): + super(OpenSSLBuilder, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + + def _build(self, install_dirs, reconfigure): + configure = os.path.join(self.src_dir, "Configure") + + # prefer to resolve the perl that we installed from + # our manifest on windows, but fall back to the system + # path on eg: darwin + env = self.env.copy() + for d in install_dirs: + bindir = os.path.join(d, "bin") + add_path_entry(env, "PATH", bindir, append=False) + + perl = path_search(env, "perl", "perl") + + if self.build_opts.is_windows(): + make = "nmake.exe" + args = ["VC-WIN64A-masm", "-utf-8"] + elif self.build_opts.is_darwin(): + make = "make" + args = ["darwin64-x86_64-cc"] + elif self.build_opts.is_linux(): + make = "make" + args = ( + ["linux-x86_64"] if not self.build_opts.is_arm() else ["linux-aarch64"] + ) + else: + raise Exception("don't know how to build openssl for %r" % self.ctx) + + self._run_cmd( + [ + perl, + configure, + "--prefix=%s" % self.inst_dir, + "--openssldir=%s" % self.inst_dir, + ] + + args + + [ + "enable-static-engine", + "enable-capieng", + "no-makedepend", + "no-unit-test", + "no-tests", + ] + ) + self._run_cmd([make, "install_sw", "install_ssldirs"]) + + +class Boost(BuilderBase): + def __init__( + self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, b2_args + ): + children = os.listdir(src_dir) + assert len(children) == 1, "expected a single directory entry: %r" % (children,) + boost_src = children[0] + assert boost_src.startswith("boost") + src_dir = os.path.join(src_dir, children[0]) + super(Boost, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + self.b2_args = b2_args + + def _build(self, install_dirs, reconfigure): + env = self._compute_env(install_dirs) + linkage = ["static"] + if self.build_opts.is_windows(): + linkage.append("shared") + + args = [] + if self.build_opts.is_darwin(): + clang = subprocess.check_output(["xcrun", "--find", "clang"]) + user_config = os.path.join(self.build_dir, "project-config.jam") + with open(user_config, "w") as jamfile: + jamfile.write("using clang : : %s ;\n" % clang.decode().strip()) + args.append("--user-config=%s" % user_config) + + for link in linkage: + if self.build_opts.is_windows(): + bootstrap = os.path.join(self.src_dir, "bootstrap.bat") + self._run_cmd([bootstrap], cwd=self.src_dir, env=env) + args += ["address-model=64"] + else: + bootstrap = os.path.join(self.src_dir, "bootstrap.sh") + self._run_cmd( + [bootstrap, "--prefix=%s" % self.inst_dir], + cwd=self.src_dir, + env=env, + ) + + b2 = os.path.join(self.src_dir, "b2") + self._run_cmd( + [ + b2, + "-j%s" % self.build_opts.num_jobs, + "--prefix=%s" % self.inst_dir, + "--builddir=%s" % self.build_dir, + ] + + args + + self.b2_args + + [ + "link=%s" % link, + "runtime-link=shared", + "variant=release", + "threading=multi", + "debug-symbols=on", + "visibility=global", + "-d2", + "install", + ], + cwd=self.src_dir, + env=env, + ) + + +class NopBuilder(BuilderBase): + def __init__(self, build_opts, ctx, manifest, src_dir, inst_dir): + super(NopBuilder, self).__init__( + build_opts, ctx, manifest, src_dir, None, inst_dir + ) + + def build(self, install_dirs, reconfigure): + print("Installing %s -> %s" % (self.src_dir, self.inst_dir)) + parent = os.path.dirname(self.inst_dir) + if not os.path.exists(parent): + os.makedirs(parent) + + install_files = self.manifest.get_section_as_ordered_pairs( + "install.files", self.ctx + ) + if install_files: + for src_name, dest_name in self.manifest.get_section_as_ordered_pairs( + "install.files", self.ctx + ): + full_dest = os.path.join(self.inst_dir, dest_name) + full_src = os.path.join(self.src_dir, src_name) + + dest_parent = os.path.dirname(full_dest) + if not os.path.exists(dest_parent): + os.makedirs(dest_parent) + if os.path.isdir(full_src): + if not os.path.exists(full_dest): + shutil.copytree(full_src, full_dest) + else: + shutil.copyfile(full_src, full_dest) + shutil.copymode(full_src, full_dest) + # This is a bit gross, but the mac ninja.zip doesn't + # give ninja execute permissions, so force them on + # for things that look like they live in a bin dir + if os.path.dirname(dest_name) == "bin": + st = os.lstat(full_dest) + os.chmod(full_dest, st.st_mode | stat.S_IXUSR) + else: + if not os.path.exists(self.inst_dir): + shutil.copytree(self.src_dir, self.inst_dir) + + +class OpenNSABuilder(NopBuilder): + # OpenNSA libraries are stored with git LFS. As a result, fetcher fetches + # LFS pointers and not the contents. Use git-lfs to pull the real contents + # before copying to install dir using NoopBuilder. + # In future, if more builders require git-lfs, we would consider installing + # git-lfs as part of the sandcastle infra as against repeating similar + # logic for each builder that requires git-lfs. + def __init__(self, build_opts, ctx, manifest, src_dir, inst_dir): + super(OpenNSABuilder, self).__init__( + build_opts, ctx, manifest, src_dir, inst_dir + ) + + def build(self, install_dirs, reconfigure): + env = self._compute_env(install_dirs) + self._run_cmd(["git", "lfs", "install", "--local"], cwd=self.src_dir, env=env) + self._run_cmd(["git", "lfs", "pull"], cwd=self.src_dir, env=env) + + super(OpenNSABuilder, self).build(install_dirs, reconfigure) + + +class SqliteBuilder(BuilderBase): + def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir): + super(SqliteBuilder, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + + def _build(self, install_dirs, reconfigure): + for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]: + src = os.path.join(self.src_dir, f) + dest = os.path.join(self.build_dir, f) + copy_if_different(src, dest) + + cmake_lists = """ +cmake_minimum_required(VERSION 3.1.3 FATAL_ERROR) +project(sqlite3 C) +add_library(sqlite3 STATIC sqlite3.c) +# These options are taken from the defaults in Makefile.msc in +# the sqlite distribution +target_compile_definitions(sqlite3 PRIVATE + -DSQLITE_ENABLE_COLUMN_METADATA=1 + -DSQLITE_ENABLE_FTS3=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_ENABLE_GEOPOLY=1 + -DSQLITE_ENABLE_JSON1=1 + -DSQLITE_ENABLE_STMTVTAB=1 + -DSQLITE_ENABLE_DBPAGE_VTAB=1 + -DSQLITE_ENABLE_DBSTAT_VTAB=1 + -DSQLITE_INTROSPECTION_PRAGMAS=1 + -DSQLITE_ENABLE_DESERIALIZE=1 +) +install(TARGETS sqlite3) +install(FILES sqlite3.h sqlite3ext.h DESTINATION include) + """ + + with open(os.path.join(self.build_dir, "CMakeLists.txt"), "w") as f: + f.write(cmake_lists) + + defines = { + "CMAKE_INSTALL_PREFIX": self.inst_dir, + "BUILD_SHARED_LIBS": "OFF", + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + } + define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()] + define_args += ["-G", "Ninja"] + + env = self._compute_env(install_dirs) + + # Resolve the cmake that we installed + cmake = path_search(env, "cmake") + + self._run_cmd([cmake, self.build_dir] + define_args, env=env) + self._run_cmd( + [ + cmake, + "--build", + self.build_dir, + "--target", + "install", + "--config", + "Release", + "-j", + str(self.build_opts.num_jobs), + ], + env=env, + ) + + +class CargoBuilder(BuilderBase): + def __init__( + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + build_doc, + workspace_dir, + manifests_to_build, + loader, + ): + super(CargoBuilder, self).__init__( + build_opts, ctx, manifest, src_dir, build_dir, inst_dir + ) + self.build_doc = build_doc + self.ws_dir = workspace_dir + self.manifests_to_build = manifests_to_build and manifests_to_build.split(",") + self.loader = loader + + def run_cargo(self, install_dirs, operation, args=None): + args = args or [] + env = self._compute_env(install_dirs) + # Enable using nightly features with stable compiler + env["RUSTC_BOOTSTRAP"] = "1" + env["LIBZ_SYS_STATIC"] = "1" + cmd = [ + "cargo", + operation, + "--workspace", + "-j%s" % self.build_opts.num_jobs, + ] + args + self._run_cmd(cmd, cwd=self.workspace_dir(), env=env) + + def build_source_dir(self): + return os.path.join(self.build_dir, "source") + + def workspace_dir(self): + return os.path.join(self.build_source_dir(), self.ws_dir or "") + + def manifest_dir(self, manifest): + return os.path.join(self.build_source_dir(), manifest) + + def recreate_dir(self, src, dst): + if os.path.isdir(dst): + shutil.rmtree(dst) + shutil.copytree(src, dst) + + def _build(self, install_dirs, reconfigure): + build_source_dir = self.build_source_dir() + self.recreate_dir(self.src_dir, build_source_dir) + + dot_cargo_dir = os.path.join(build_source_dir, ".cargo") + if not os.path.isdir(dot_cargo_dir): + os.mkdir(dot_cargo_dir) + + with open(os.path.join(dot_cargo_dir, "config"), "w+") as f: + f.write( + """\ +[build] +target-dir = '''{}''' + +[net] +git-fetch-with-cli = true + +[profile.dev] +debug = false +incremental = false +""".format( + self.build_dir.replace("\\", "\\\\") + ) + ) + + if self.ws_dir is not None: + self._patchup_workspace() + + try: + from getdeps.facebook.rust import vendored_crates + + vendored_crates(self.build_opts, build_source_dir) + except ImportError: + # This FB internal module isn't shippped to github, + # so just rely on cargo downloading crates on it's own + pass + + if self.manifests_to_build is None: + self.run_cargo( + install_dirs, + "build", + ["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"], + ) + else: + for manifest in self.manifests_to_build: + self.run_cargo( + install_dirs, + "build", + [ + "--out-dir", + os.path.join(self.inst_dir, "bin"), + "-Zunstable-options", + "--manifest-path", + self.manifest_dir(manifest), + ], + ) + + self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source")) + + def run_tests( + self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot + ): + if test_filter: + args = ["--", test_filter] + else: + args = [] + + if self.manifests_to_build is None: + self.run_cargo(install_dirs, "test", args) + if self.build_doc: + self.run_cargo(install_dirs, "doc", ["--no-deps"]) + else: + for manifest in self.manifests_to_build: + margs = ["--manifest-path", self.manifest_dir(manifest)] + self.run_cargo(install_dirs, "test", args + margs) + if self.build_doc: + self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs) + + def _patchup_workspace(self): + """ + This method makes some assumptions about the state of the project and + its cargo dependendies: + 1. Crates from cargo dependencies can be extracted from Cargo.toml files + using _extract_crates function. It is using a heuristic so check its + code to understand how it is done. + 2. The extracted cargo dependencies crates can be found in the + dependency's install dir using _resolve_crate_to_path function + which again is using a heuristic. + + Notice that many things might go wrong here. E.g. if someone depends + on another getdeps crate by writing in their Cargo.toml file: + + my-rename-of-crate = { package = "crate", git = "..." } + + they can count themselves lucky because the code will raise an + Exception. There migh be more cases where the code will silently pass + producing bad results. + """ + workspace_dir = self.workspace_dir() + config = self._resolve_config() + if config: + with open(os.path.join(workspace_dir, "Cargo.toml"), "r+") as f: + manifest_content = f.read() + if "[package]" not in manifest_content: + # A fake manifest has to be crated to change the virtual + # manifest into a non-virtual. The virtual manifests are limited + # in many ways and the inability to define patches on them is + # one. Check https://github.com/rust-lang/cargo/issues/4934 to + # see if it is resolved. + f.write( + """ + [package] + name = "fake_manifest_of_{}" + version = "0.0.0" + [lib] + path = "/dev/null" + """.format( + self.manifest.name + ) + ) + else: + f.write("\n") + f.write(config) + + def _resolve_config(self): + """ + Returns a configuration to be put inside root Cargo.toml file which + patches the dependencies git code with local getdeps versions. + See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section + """ + dep_to_git = self._resolve_dep_to_git() + dep_to_crates = CargoBuilder._resolve_dep_to_crates( + self.build_source_dir(), dep_to_git + ) + + config = [] + for name in sorted(dep_to_git.keys()): + git_conf = dep_to_git[name] + crates = sorted(dep_to_crates.get(name, [])) + if not crates: + continue # nothing to patch, move along + crates_patches = [ + '{} = {{ path = "{}" }}'.format( + crate, + CargoBuilder._resolve_crate_to_path(crate, git_conf).replace( + "\\", "\\\\" + ), + ) + for crate in crates + ] + + config.append( + '[patch."{0}"]\n'.format(git_conf["repo_url"]) + + "\n".join(crates_patches) + ) + return "\n".join(config) + + def _resolve_dep_to_git(self): + """ + For each direct dependency of the currently build manifest check if it + is also cargo-builded and if yes then extract it's git configs and + install dir + """ + dependencies = self.manifest.get_section_as_dict("dependencies", ctx=self.ctx) + if not dependencies: + return [] + + dep_to_git = {} + for dep in dependencies.keys(): + dep_manifest = self.loader.load_manifest(dep) + dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx) + if dep_builder not in ["cargo", "nop"] or dep == "rust": + # This is a direct dependency, but it is not build with cargo + # and it is not simply copying files with nop, so ignore it. + # The "rust" dependency is an exception since it contains the + # toolchain. + continue + + git_conf = dep_manifest.get_section_as_dict("git", ctx=self.ctx) + if "repo_url" not in git_conf: + raise Exception( + "A cargo dependency requires git.repo_url to be defined." + ) + source_dir = self.loader.get_project_install_dir(dep_manifest) + if dep_builder == "cargo": + source_dir = os.path.join(source_dir, "source") + git_conf["source_dir"] = source_dir + dep_to_git[dep] = git_conf + return dep_to_git + + @staticmethod + def _resolve_dep_to_crates(build_source_dir, dep_to_git): + """ + This function traverse the build_source_dir in search of Cargo.toml + files, extracts the crate names from them using _extract_crates + function and returns a merged result containing crate names per + dependency name from all Cargo.toml files in the project. + """ + if not dep_to_git: + return {} # no deps, so don't waste time traversing files + + dep_to_crates = {} + for root, _, files in os.walk(build_source_dir): + for f in files: + if f == "Cargo.toml": + more_dep_to_crates = CargoBuilder._extract_crates( + os.path.join(root, f), dep_to_git + ) + for name, crates in more_dep_to_crates.items(): + dep_to_crates.setdefault(name, set()).update(crates) + return dep_to_crates + + @staticmethod + def _extract_crates(cargo_toml_file, dep_to_git): + """ + This functions reads content of provided cargo toml file and extracts + crate names per each dependency. The extraction is done by a heuristic + so it might be incorrect. + """ + deps_to_crates = {} + with open(cargo_toml_file, "r") as f: + for line in f.readlines(): + if line.startswith("#") or "git = " not in line: + continue # filter out commented lines and ones without git deps + for name, conf in dep_to_git.items(): + if 'git = "{}"'.format(conf["repo_url"]) in line: + pkg_template = ' package = "' + if pkg_template in line: + crate_name, _, _ = line.partition(pkg_template)[ + 2 + ].partition('"') + else: + crate_name, _, _ = line.partition("=") + deps_to_crates.setdefault(name, set()).add(crate_name.strip()) + return deps_to_crates + + @staticmethod + def _resolve_crate_to_path(crate, git_conf): + """ + Tries to find in git_conf["inst_dir"] by searching a [package] + keyword followed by name = "". + """ + source_dir = git_conf["source_dir"] + search_pattern = '[package]\nname = "{}"'.format(crate) + + for root, _, files in os.walk(source_dir): + for fname in files: + if fname == "Cargo.toml": + with open(os.path.join(root, fname), "r") as f: + if search_pattern in f.read(): + return root + + raise Exception("Failed to found crate {} in path {}".format(crate, source_dir)) diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py new file mode 100644 index 000000000000..bc6d2da87585 --- /dev/null +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -0,0 +1,458 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import errno +import glob +import ntpath +import os +import subprocess +import sys +import tempfile + +from .copytree import containing_repo_type +from .envfuncs import Env, add_path_entry +from .fetcher import get_fbsource_repo_data +from .manifest import ContextGenerator +from .platform import HostType, is_windows + + +try: + import typing # noqa: F401 +except ImportError: + pass + + +def detect_project(path): + repo_type, repo_root = containing_repo_type(path) + if repo_type is None: + return None, None + + # Look for a .projectid file. If it exists, read the project name from it. + project_id_path = os.path.join(repo_root, ".projectid") + try: + with open(project_id_path, "r") as f: + project_name = f.read().strip() + return repo_root, project_name + except EnvironmentError as ex: + if ex.errno != errno.ENOENT: + raise + + return repo_root, None + + +class BuildOptions(object): + def __init__( + self, + fbcode_builder_dir, + scratch_dir, + host_type, + install_dir=None, + num_jobs=0, + use_shipit=False, + vcvars_path=None, + allow_system_packages=False, + lfs_path=None, + ): + """fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir, + or for shipit-transformed repos, the build dir that + has been mapped into that dir. + scratch_dir - a place where we can store repos and build bits. + This path should be stable across runs and ideally + should not be in the repo of the project being built, + but that is ultimately where we generally fall back + for builds outside of FB + install_dir - where the project will ultimately be installed + num_jobs - the level of concurrency to use while building + use_shipit - use real shipit instead of the simple shipit transformer + vcvars_path - Path to external VS toolchain's vsvarsall.bat + """ + if not num_jobs: + import multiprocessing + + num_jobs = multiprocessing.cpu_count() // 2 + + if not install_dir: + install_dir = os.path.join(scratch_dir, "installed") + + self.project_hashes = None + for p in ["../deps/github_hashes", "../project_hashes"]: + hashes = os.path.join(fbcode_builder_dir, p) + if os.path.exists(hashes): + self.project_hashes = hashes + break + + # Detect what repository and project we are being run from. + self.repo_root, self.repo_project = detect_project(os.getcwd()) + + # If we are running from an fbsource repository, set self.fbsource_dir + # to allow the ShipIt-based fetchers to use it. + if self.repo_project == "fbsource": + self.fbsource_dir = self.repo_root + else: + self.fbsource_dir = None + + self.num_jobs = num_jobs + self.scratch_dir = scratch_dir + self.install_dir = install_dir + self.fbcode_builder_dir = fbcode_builder_dir + self.host_type = host_type + self.use_shipit = use_shipit + self.allow_system_packages = allow_system_packages + self.lfs_path = lfs_path + if vcvars_path is None and is_windows(): + + # On Windows, the compiler is not available in the PATH by + # default so we need to run the vcvarsall script to populate the + # environment. We use a glob to find some version of this script + # as deployed with Visual Studio 2017. This logic can also + # locate Visual Studio 2019 but note that at the time of writing + # the version of boost in our manifest cannot be built with + # VS 2019, so we're effectively tied to VS 2017 until we upgrade + # the boost dependency. + vcvarsall = [] + for year in ["2017", "2019"]: + vcvarsall += glob.glob( + os.path.join( + os.environ["ProgramFiles(x86)"], + "Microsoft Visual Studio", + year, + "*", + "VC", + "Auxiliary", + "Build", + "vcvarsall.bat", + ) + ) + vcvars_path = vcvarsall[0] + + self.vcvars_path = vcvars_path + + @property + def manifests_dir(self): + return os.path.join(self.fbcode_builder_dir, "manifests") + + def is_darwin(self): + return self.host_type.is_darwin() + + def is_windows(self): + return self.host_type.is_windows() + + def is_arm(self): + return self.host_type.is_arm() + + def get_vcvars_path(self): + return self.vcvars_path + + def is_linux(self): + return self.host_type.is_linux() + + def get_context_generator(self, host_tuple=None, facebook_internal=None): + """Create a manifest ContextGenerator for the specified target platform.""" + if host_tuple is None: + host_type = self.host_type + elif isinstance(host_tuple, HostType): + host_type = host_tuple + else: + host_type = HostType.from_tuple_string(host_tuple) + + # facebook_internal is an Optional[bool] + # If it is None, default to assuming this is a Facebook-internal build if + # we are running in an fbsource repository. + if facebook_internal is None: + facebook_internal = self.fbsource_dir is not None + + return ContextGenerator( + { + "os": host_type.ostype, + "distro": host_type.distro, + "distro_vers": host_type.distrovers, + "fb": "on" if facebook_internal else "off", + "test": "off", + } + ) + + def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None): + if env is not None: + env = env.copy() + else: + env = Env() + + env["GETDEPS_BUILD_DIR"] = os.path.join(self.scratch_dir, "build") + env["GETDEPS_INSTALL_DIR"] = self.install_dir + + # On macOS we need to set `SDKROOT` when we use clang for system + # header files. + if self.is_darwin() and "SDKROOT" not in env: + sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"]) + env["SDKROOT"] = sdkroot.decode().strip() + + if self.fbsource_dir: + env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join( + self.fbsource_dir, "xplat/third-party/yarn/offline-mirror" + ) + yarn_exe = "yarn.bat" if self.is_windows() else "yarn" + env["YARN_PATH"] = os.path.join( + self.fbsource_dir, "xplat/third-party/yarn/", yarn_exe + ) + node_exe = "node-win-x64.exe" if self.is_windows() else "node" + env["NODE_BIN"] = os.path.join( + self.fbsource_dir, "xplat/third-party/node/bin/", node_exe + ) + env["RUST_VENDORED_CRATES_DIR"] = os.path.join( + self.fbsource_dir, "third-party/rust/vendor" + ) + hash_data = get_fbsource_repo_data(self) + env["FBSOURCE_HASH"] = hash_data.hash + env["FBSOURCE_DATE"] = hash_data.date + + lib_path = None + if self.is_darwin(): + lib_path = "DYLD_LIBRARY_PATH" + elif self.is_linux(): + lib_path = "LD_LIBRARY_PATH" + elif self.is_windows(): + lib_path = "PATH" + else: + lib_path = None + + for d in install_dirs: + bindir = os.path.join(d, "bin") + + if not ( + manifest and manifest.get("build", "disable_env_override_pkgconfig") + ): + pkgconfig = os.path.join(d, "lib/pkgconfig") + if os.path.exists(pkgconfig): + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + + pkgconfig = os.path.join(d, "lib64/pkgconfig") + if os.path.exists(pkgconfig): + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + + if not (manifest and manifest.get("build", "disable_env_override_path")): + add_path_entry(env, "CMAKE_PREFIX_PATH", d) + + # Allow resolving shared objects built earlier (eg: zstd + # doesn't include the full path to the dylib in its linkage + # so we need to give it an assist) + if lib_path: + for lib in ["lib", "lib64"]: + libdir = os.path.join(d, lib) + if os.path.exists(libdir): + add_path_entry(env, lib_path, libdir) + + # Allow resolving binaries (eg: cmake, ninja) and dlls + # built by earlier steps + if os.path.exists(bindir): + add_path_entry(env, "PATH", bindir, append=False) + + # If rustc is present in the `bin` directory, set RUSTC to prevent + # cargo uses the rustc installed in the system. + if self.is_windows(): + cargo_path = os.path.join(bindir, "cargo.exe") + rustc_path = os.path.join(bindir, "rustc.exe") + rustdoc_path = os.path.join(bindir, "rustdoc.exe") + else: + cargo_path = os.path.join(bindir, "cargo") + rustc_path = os.path.join(bindir, "rustc") + rustdoc_path = os.path.join(bindir, "rustdoc") + + if os.path.isfile(rustc_path): + env["CARGO_BIN"] = cargo_path + env["RUSTC"] = rustc_path + env["RUSTDOC"] = rustdoc_path + + openssl_include = os.path.join(d, "include/openssl") + if os.path.isdir(openssl_include) and any( + os.path.isfile(os.path.join(d, "lib", libcrypto)) + for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a") + ): + # This must be the openssl library, let Rust know about it + env["OPENSSL_DIR"] = d + + return env + + +def list_win32_subst_letters(): + output = subprocess.check_output(["subst"]).decode("utf-8") + # The output is a set of lines like: `F:\: => C:\open\some\where` + lines = output.strip().split("\r\n") + mapping = {} + for line in lines: + fields = line.split(": => ") + if len(fields) != 2: + continue + letter = fields[0] + path = fields[1] + mapping[letter] = path + + return mapping + + +def find_existing_win32_subst_for_path( + path, # type: str + subst_mapping, # type: typing.Mapping[str, str] +): + # type: (...) -> typing.Optional[str] + path = ntpath.normcase(ntpath.normpath(path)) + for letter, target in subst_mapping.items(): + if ntpath.normcase(target) == path: + return letter + return None + + +def find_unused_drive_letter(): + import ctypes + + buffer_len = 256 + blen = ctypes.c_uint(buffer_len) + rv = ctypes.c_uint() + bufs = ctypes.create_string_buffer(buffer_len) + rv = ctypes.windll.kernel32.GetLogicalDriveStringsA(blen, bufs) + if rv > buffer_len: + raise Exception("GetLogicalDriveStringsA result too large for buffer") + nul = "\x00".encode("ascii") + + used = [drive.decode("ascii")[0] for drive in bufs.raw.strip(nul).split(nul)] + possible = [c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"] + available = sorted(list(set(possible) - set(used))) + if len(available) == 0: + return None + # Prefer to assign later letters rather than earlier letters + return available[-1] + + +def create_subst_path(path): + for _attempt in range(0, 24): + drive = find_existing_win32_subst_for_path( + path, subst_mapping=list_win32_subst_letters() + ) + if drive: + return drive + available = find_unused_drive_letter() + if available is None: + raise Exception( + ( + "unable to make shorter subst mapping for %s; " + "no available drive letters" + ) + % path + ) + + # Try to set up a subst mapping; note that we may be racing with + # other processes on the same host, so this may not succeed. + try: + subprocess.check_call(["subst", "%s:" % available, path]) + return "%s:\\" % available + except Exception: + print("Failed to map %s -> %s" % (available, path)) + + raise Exception("failed to set up a subst path for %s" % path) + + +def _check_host_type(args, host_type): + if host_type is None: + host_tuple_string = getattr(args, "host_type", None) + if host_tuple_string: + host_type = HostType.from_tuple_string(host_tuple_string) + else: + host_type = HostType() + + assert isinstance(host_type, HostType) + return host_type + + +def setup_build_options(args, host_type=None): + """Create a BuildOptions object based on the arguments""" + + fbcode_builder_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + scratch_dir = args.scratch_path + if not scratch_dir: + # TODO: `mkscratch` doesn't currently know how best to place things on + # sandcastle, so whip up something reasonable-ish + if "SANDCASTLE" in os.environ: + if "DISK_TEMP" not in os.environ: + raise Exception( + ( + "I need DISK_TEMP to be set in the sandcastle environment " + "so that I can store build products somewhere sane" + ) + ) + scratch_dir = os.path.join( + os.environ["DISK_TEMP"], "fbcode_builder_getdeps" + ) + if not scratch_dir: + try: + scratch_dir = ( + subprocess.check_output( + ["mkscratch", "path", "--subdir", "fbcode_builder_getdeps"] + ) + .strip() + .decode("utf-8") + ) + except OSError as exc: + if exc.errno != errno.ENOENT: + # A legit failure; don't fall back, surface the error + raise + # This system doesn't have mkscratch so we fall back to + # something local. + munged = fbcode_builder_dir.replace("Z", "zZ") + for s in ["/", "\\", ":"]: + munged = munged.replace(s, "Z") + + if is_windows() and os.path.isdir("c:/open"): + temp = "c:/open/scratch" + else: + temp = tempfile.gettempdir() + + scratch_dir = os.path.join(temp, "fbcode_builder_getdeps-%s" % munged) + if not is_windows() and os.geteuid() == 0: + # Running as root; in the case where someone runs + # sudo getdeps.py install-system-deps + # and then runs as build without privs, we want to avoid creating + # a scratch dir that the second stage cannot write to. + # So we generate a different path if we are root. + scratch_dir += "-root" + + if not os.path.exists(scratch_dir): + os.makedirs(scratch_dir) + + if is_windows(): + subst = create_subst_path(scratch_dir) + print( + "Mapping scratch dir %s -> %s" % (scratch_dir, subst), file=sys.stderr + ) + scratch_dir = subst + else: + if not os.path.exists(scratch_dir): + os.makedirs(scratch_dir) + + # Make sure we normalize the scratch path. This path is used as part of the hash + # computation for detecting if projects have been updated, so we need to always + # use the exact same string to refer to a given directory. + # But! realpath in some combinations of Windows/Python3 versions can expand the + # drive substitutions on Windows, so avoid that! + if not is_windows(): + scratch_dir = os.path.realpath(scratch_dir) + + # Save any extra cmake defines passed by the user in an env variable, so it + # can be used while hashing this build. + os.environ["GETDEPS_CMAKE_DEFINES"] = getattr(args, "extra_cmake_defines", "") or "" + + host_type = _check_host_type(args, host_type) + + return BuildOptions( + fbcode_builder_dir, + scratch_dir, + host_type, + install_dir=args.install_prefix, + num_jobs=args.num_jobs, + use_shipit=args.use_shipit, + vcvars_path=args.vcvars_path, + allow_system_packages=args.allow_system_packages, + lfs_path=args.lfs_path, + ) diff --git a/build/fbcode_builder/getdeps/cache.py b/build/fbcode_builder/getdeps/cache.py new file mode 100644 index 000000000000..a261541c7461 --- /dev/null +++ b/build/fbcode_builder/getdeps/cache.py @@ -0,0 +1,39 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + + +class ArtifactCache(object): + """The ArtifactCache is a small abstraction that allows caching + named things in some external storage mechanism. + The primary use case is for storing the build products on CI + systems to accelerate the build""" + + def download_to_file(self, name, dest_file_name): + """If `name` exists in the cache, download it and place it + in the specified `dest_file_name` location on the filesystem. + If a transient issue was encountered a TransientFailure shall + be raised. + If `name` doesn't exist in the cache `False` shall be returned. + If `dest_file_name` was successfully updated `True` shall be + returned. + All other conditions shall raise an appropriate exception.""" + return False + + def upload_from_file(self, name, source_file_name): + """Causes `name` to be populated in the cache by uploading + the contents of `source_file_name` to the storage system. + If a transient issue was encountered a TransientFailure shall + be raised. + If the upload failed for some other reason, an appropriate + exception shall be raised.""" + pass + + +def create_cache(): + """This function is monkey patchable to provide an actual + implementation""" + return None diff --git a/build/fbcode_builder/getdeps/copytree.py b/build/fbcode_builder/getdeps/copytree.py new file mode 100644 index 000000000000..2790bc0d92d1 --- /dev/null +++ b/build/fbcode_builder/getdeps/copytree.py @@ -0,0 +1,78 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import shutil +import subprocess + +from .platform import is_windows + + +PREFETCHED_DIRS = set() + + +def containing_repo_type(path): + while True: + if os.path.exists(os.path.join(path, ".git")): + return ("git", path) + if os.path.exists(os.path.join(path, ".hg")): + return ("hg", path) + + parent = os.path.dirname(path) + if parent == path: + return None, None + path = parent + + +def find_eden_root(dirpath): + """If the specified directory is inside an EdenFS checkout, returns + the canonical absolute path to the root of that checkout. + + Returns None if the specified directory is not in an EdenFS checkout. + """ + if is_windows(): + repo_type, repo_root = containing_repo_type(dirpath) + if repo_root is not None: + if os.path.exists(os.path.join(repo_root, ".eden", "config")): + return os.path.realpath(repo_root) + return None + + try: + return os.readlink(os.path.join(dirpath, ".eden", "root")) + except OSError: + return None + + +def prefetch_dir_if_eden(dirpath): + """After an amend/rebase, Eden may need to fetch a large number + of trees from the servers. The simplistic single threaded walk + performed by copytree makes this more expensive than is desirable + so we help accelerate things by performing a prefetch on the + source directory""" + global PREFETCHED_DIRS + if dirpath in PREFETCHED_DIRS: + return + root = find_eden_root(dirpath) + if root is None: + return + glob = f"{os.path.relpath(dirpath, root).replace(os.sep, '/')}/**" + print(f"Prefetching {glob}") + subprocess.call(["edenfsctl", "prefetch", "--repo", root, "--silent", glob]) + PREFETCHED_DIRS.add(dirpath) + + +def copytree(src_dir, dest_dir, ignore=None): + """Recursively copy the src_dir to the dest_dir, filtering + out entries using the ignore lambda. The behavior of the + ignore lambda must match that described by `shutil.copytree`. + This `copytree` function knows how to prefetch data when + running in an eden repo. + TODO: I'd like to either extend this or add a variant that + uses watchman to mirror src_dir into dest_dir. + """ + prefetch_dir_if_eden(src_dir) + return shutil.copytree(src_dir, dest_dir, ignore=ignore) diff --git a/build/fbcode_builder/getdeps/dyndeps.py b/build/fbcode_builder/getdeps/dyndeps.py new file mode 100644 index 000000000000..216f26c46d27 --- /dev/null +++ b/build/fbcode_builder/getdeps/dyndeps.py @@ -0,0 +1,430 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import errno +import glob +import os +import re +import shutil +import stat +import subprocess +import sys +from struct import unpack + +from .envfuncs import path_search + + +OBJECT_SUBDIRS = ("bin", "lib", "lib64") + + +def copyfile(src, dest): + shutil.copyfile(src, dest) + shutil.copymode(src, dest) + + +class DepBase(object): + def __init__(self, buildopts, install_dirs, strip): + self.buildopts = buildopts + self.env = buildopts.compute_env_for_install_dirs(install_dirs) + self.install_dirs = install_dirs + self.strip = strip + self.processed_deps = set() + + def list_dynamic_deps(self, objfile): + raise RuntimeError("list_dynamic_deps not implemented") + + def interesting_dep(self, d): + return True + + # final_install_prefix must be the equivalent path to `destdir` on the + # installed system. For example, if destdir is `/tmp/RANDOM/usr/local' which + # is intended to map to `/usr/local` in the install image, then + # final_install_prefix='/usr/local'. + # If left unspecified, destdir will be used. + def process_deps(self, destdir, final_install_prefix=None): + if self.buildopts.is_windows(): + lib_dir = "bin" + else: + lib_dir = "lib" + self.munged_lib_dir = os.path.join(destdir, lib_dir) + + final_lib_dir = os.path.join(final_install_prefix or destdir, lib_dir) + + if not os.path.isdir(self.munged_lib_dir): + os.makedirs(self.munged_lib_dir) + + # Look only at the things that got installed in the leaf package, + # which will be the last entry in the install dirs list + inst_dir = self.install_dirs[-1] + print("Process deps under %s" % inst_dir, file=sys.stderr) + + for dir in OBJECT_SUBDIRS: + src_dir = os.path.join(inst_dir, dir) + if not os.path.isdir(src_dir): + continue + dest_dir = os.path.join(destdir, dir) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + + for objfile in self.list_objs_in_dir(src_dir): + print("Consider %s/%s" % (dir, objfile)) + dest_obj = os.path.join(dest_dir, objfile) + copyfile(os.path.join(src_dir, objfile), dest_obj) + self.munge_in_place(dest_obj, final_lib_dir) + + def find_all_dependencies(self, build_dir): + all_deps = set() + for objfile in self.list_objs_in_dir( + build_dir, recurse=True, output_prefix=build_dir + ): + for d in self.list_dynamic_deps(objfile): + all_deps.add(d) + + interesting_deps = {d for d in all_deps if self.interesting_dep(d)} + dep_paths = [] + for dep in interesting_deps: + dep_path = self.resolve_loader_path(dep) + if dep_path: + dep_paths.append(dep_path) + + return dep_paths + + def munge_in_place(self, objfile, final_lib_dir): + print("Munging %s" % objfile) + for d in self.list_dynamic_deps(objfile): + if not self.interesting_dep(d): + continue + + # Resolve this dep: does it exist in any of our installation + # directories? If so, then it is a candidate for processing + dep = self.resolve_loader_path(d) + print("dep: %s -> %s" % (d, dep)) + if dep: + dest_dep = os.path.join(self.munged_lib_dir, os.path.basename(dep)) + if dep not in self.processed_deps: + self.processed_deps.add(dep) + copyfile(dep, dest_dep) + self.munge_in_place(dest_dep, final_lib_dir) + + self.rewrite_dep(objfile, d, dep, dest_dep, final_lib_dir) + + if self.strip: + self.strip_debug_info(objfile) + + def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir): + raise RuntimeError("rewrite_dep not implemented") + + def resolve_loader_path(self, dep): + if os.path.isabs(dep): + return dep + d = os.path.basename(dep) + for inst_dir in self.install_dirs: + for libdir in OBJECT_SUBDIRS: + candidate = os.path.join(inst_dir, libdir, d) + if os.path.exists(candidate): + return candidate + return None + + def list_objs_in_dir(self, dir, recurse=False, output_prefix=""): + for entry in os.listdir(dir): + entry_path = os.path.join(dir, entry) + st = os.lstat(entry_path) + if stat.S_ISREG(st.st_mode): + if self.is_objfile(entry_path): + relative_result = os.path.join(output_prefix, entry) + yield os.path.normcase(relative_result) + elif recurse and stat.S_ISDIR(st.st_mode): + child_prefix = os.path.join(output_prefix, entry) + for result in self.list_objs_in_dir( + entry_path, recurse=recurse, output_prefix=child_prefix + ): + yield result + + def is_objfile(self, objfile): + return True + + def strip_debug_info(self, objfile): + """override this to define how to remove debug information + from an object file""" + pass + + +class WinDeps(DepBase): + def __init__(self, buildopts, install_dirs, strip): + super(WinDeps, self).__init__(buildopts, install_dirs, strip) + self.dumpbin = self.find_dumpbin() + + def find_dumpbin(self): + # Looking for dumpbin in the following hardcoded paths. + # The registry option to find the install dir doesn't work anymore. + globs = [ + ( + "C:/Program Files (x86)/" + "Microsoft Visual Studio/" + "*/*/VC/Tools/" + "MSVC/*/bin/Hostx64/x64/dumpbin.exe" + ), + ( + "C:/Program Files (x86)/" + "Common Files/" + "Microsoft/Visual C++ for Python/*/" + "VC/bin/dumpbin.exe" + ), + ("c:/Program Files (x86)/Microsoft Visual Studio */VC/bin/dumpbin.exe"), + ] + for pattern in globs: + for exe in glob.glob(pattern): + return exe + + raise RuntimeError("could not find dumpbin.exe") + + def list_dynamic_deps(self, exe): + deps = [] + print("Resolve deps for %s" % exe) + output = subprocess.check_output( + [self.dumpbin, "/nologo", "/dependents", exe] + ).decode("utf-8") + + lines = output.split("\n") + for line in lines: + m = re.match("\\s+(\\S+.dll)", line, re.IGNORECASE) + if m: + deps.append(m.group(1).lower()) + + return deps + + def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir): + # We can't rewrite on windows, but we will + # place the deps alongside the exe so that + # they end up in the search path + pass + + # These are the Windows system dll, which we don't want to copy while + # packaging. + SYSTEM_DLLS = set( # noqa: C405 + [ + "advapi32.dll", + "dbghelp.dll", + "kernel32.dll", + "msvcp140.dll", + "vcruntime140.dll", + "ws2_32.dll", + "ntdll.dll", + "shlwapi.dll", + ] + ) + + def interesting_dep(self, d): + if "api-ms-win-crt" in d: + return False + if d in self.SYSTEM_DLLS: + return False + return True + + def is_objfile(self, objfile): + if not os.path.isfile(objfile): + return False + if objfile.lower().endswith(".exe"): + return True + return False + + def emit_dev_run_script(self, script_path, dep_dirs): + """Emit a script that can be used to run build artifacts directly from the + build directory, without installing them. + + The dep_dirs parameter should be a list of paths that need to be added to $PATH. + This can be computed by calling compute_dependency_paths() or + compute_dependency_paths_fast(). + + This is only necessary on Windows, which does not have RPATH, and instead + requires the $PATH environment variable be updated in order to find the proper + library dependencies. + """ + contents = self._get_dev_run_script_contents(dep_dirs) + with open(script_path, "w") as f: + f.write(contents) + + def compute_dependency_paths(self, build_dir): + """Return a list of all directories that need to be added to $PATH to ensure + that library dependencies can be found correctly. This is computed by scanning + binaries to determine exactly the right list of dependencies. + + The compute_dependency_paths_fast() is a alternative function that runs faster + but may return additional extraneous paths. + """ + dep_dirs = set() + # Find paths by scanning the binaries. + for dep in self.find_all_dependencies(build_dir): + dep_dirs.add(os.path.dirname(dep)) + + dep_dirs.update(self.read_custom_dep_dirs(build_dir)) + return sorted(dep_dirs) + + def compute_dependency_paths_fast(self, build_dir): + """Similar to compute_dependency_paths(), but rather than actually scanning + binaries, just add all library paths from the specified installation + directories. This is much faster than scanning the binaries, but may result in + more paths being returned than actually necessary. + """ + dep_dirs = set() + for inst_dir in self.install_dirs: + for subdir in OBJECT_SUBDIRS: + path = os.path.join(inst_dir, subdir) + if os.path.exists(path): + dep_dirs.add(path) + + dep_dirs.update(self.read_custom_dep_dirs(build_dir)) + return sorted(dep_dirs) + + def read_custom_dep_dirs(self, build_dir): + # The build system may also have included libraries from other locations that + # we might not be able to find normally in find_all_dependencies(). + # To handle this situation we support reading additional library paths + # from a LIBRARY_DEP_DIRS.txt file that may have been generated in the build + # output directory. + dep_dirs = set() + try: + explicit_dep_dirs_path = os.path.join(build_dir, "LIBRARY_DEP_DIRS.txt") + with open(explicit_dep_dirs_path, "r") as f: + for line in f.read().splitlines(): + dep_dirs.add(line) + except OSError as ex: + if ex.errno != errno.ENOENT: + raise + + return dep_dirs + + def _get_dev_run_script_contents(self, path_dirs): + path_entries = ["$env:PATH"] + path_dirs + path_str = ";".join(path_entries) + return """\ +$orig_env = $env:PATH +$env:PATH = "{path_str}" + +try {{ + $cmd_args = $args[1..$args.length] + & $args[0] @cmd_args +}} finally {{ + $env:PATH = $orig_env +}} +""".format( + path_str=path_str + ) + + +class ElfDeps(DepBase): + def __init__(self, buildopts, install_dirs, strip): + super(ElfDeps, self).__init__(buildopts, install_dirs, strip) + + # We need patchelf to rewrite deps, so ensure that it is built... + subprocess.check_call([sys.executable, sys.argv[0], "build", "patchelf"]) + # ... and that we know where it lives + self.patchelf = os.path.join( + os.fsdecode( + subprocess.check_output( + [sys.executable, sys.argv[0], "show-inst-dir", "patchelf"] + ).strip() + ), + "bin/patchelf", + ) + + def list_dynamic_deps(self, objfile): + out = ( + subprocess.check_output( + [self.patchelf, "--print-needed", objfile], env=dict(self.env.items()) + ) + .decode("utf-8") + .strip() + ) + lines = out.split("\n") + return lines + + def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir): + final_dep = os.path.join( + final_lib_dir, os.path.relpath(new_dep, self.munged_lib_dir) + ) + subprocess.check_call( + [self.patchelf, "--replace-needed", depname, final_dep, objfile] + ) + + def is_objfile(self, objfile): + if not os.path.isfile(objfile): + return False + with open(objfile, "rb") as f: + # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header + magic = f.read(4) + return magic == b"\x7fELF" + + def strip_debug_info(self, objfile): + subprocess.check_call(["strip", objfile]) + + +# MACH-O magic number +MACH_MAGIC = 0xFEEDFACF + + +class MachDeps(DepBase): + def interesting_dep(self, d): + if d.startswith("/usr/lib/") or d.startswith("/System/"): + return False + return True + + def is_objfile(self, objfile): + if not os.path.isfile(objfile): + return False + with open(objfile, "rb") as f: + # mach stores the magic number in native endianness, + # so unpack as native here and compare + header = f.read(4) + if len(header) != 4: + return False + magic = unpack("I", header)[0] + return magic == MACH_MAGIC + + def list_dynamic_deps(self, objfile): + if not self.interesting_dep(objfile): + return + out = ( + subprocess.check_output( + ["otool", "-L", objfile], env=dict(self.env.items()) + ) + .decode("utf-8") + .strip() + ) + lines = out.split("\n") + deps = [] + for line in lines: + m = re.match("\t(\\S+)\\s", line) + if m: + if os.path.basename(m.group(1)) != os.path.basename(objfile): + deps.append(os.path.normcase(m.group(1))) + return deps + + def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir): + if objfile.endswith(".dylib"): + # Erase the original location from the id of the shared + # object. It doesn't appear to hurt to retain it, but + # it does look weird, so let's rewrite it to be sure. + subprocess.check_call( + ["install_name_tool", "-id", os.path.basename(objfile), objfile] + ) + final_dep = os.path.join( + final_lib_dir, os.path.relpath(new_dep, self.munged_lib_dir) + ) + + subprocess.check_call( + ["install_name_tool", "-change", depname, final_dep, objfile] + ) + + +def create_dyn_dep_munger(buildopts, install_dirs, strip=False): + if buildopts.is_linux(): + return ElfDeps(buildopts, install_dirs, strip) + if buildopts.is_darwin(): + return MachDeps(buildopts, install_dirs, strip) + if buildopts.is_windows(): + return WinDeps(buildopts, install_dirs, strip) diff --git a/build/fbcode_builder/getdeps/envfuncs.py b/build/fbcode_builder/getdeps/envfuncs.py new file mode 100644 index 000000000000..f2e13f16fafa --- /dev/null +++ b/build/fbcode_builder/getdeps/envfuncs.py @@ -0,0 +1,195 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import shlex +import sys + + +class Env(object): + def __init__(self, src=None): + self._dict = {} + if src is None: + self.update(os.environ) + else: + self.update(src) + + def update(self, src): + for k, v in src.items(): + self.set(k, v) + + def copy(self): + return Env(self._dict) + + def _key(self, key): + # The `str` cast may not appear to be needed, but without it we run + # into issues when passing the environment to subprocess. The main + # issue is that in python2 `os.environ` (which is the initial source + # of data for the environment) uses byte based strings, but this + # project uses `unicode_literals`. `subprocess` will raise an error + # if the environment that it is passed has a mixture of byte and + # unicode strings. + # It is simplest to force everthing to be `str` for the sake of + # consistency. + key = str(key) + if sys.platform.startswith("win"): + # Windows env var names are case insensitive but case preserving. + # An implementation of PAR files on windows gets confused if + # the env block contains keys with conflicting case, so make a + # pass over the contents to remove any. + # While this O(n) scan is technically expensive and gross, it + # is practically not a problem because the volume of calls is + # relatively low and the cost of manipulating the env is dwarfed + # by the cost of spawning a process on windows. In addition, + # since the processes that we run are expensive anyway, this + # overhead is not the worst thing to worry about. + for k in list(self._dict.keys()): + if str(k).lower() == key.lower(): + return k + elif key in self._dict: + return key + return None + + def get(self, key, defval=None): + key = self._key(key) + if key is None: + return defval + return self._dict[key] + + def __getitem__(self, key): + val = self.get(key) + if key is None: + raise KeyError(key) + return val + + def unset(self, key): + if key is None: + raise KeyError("attempting to unset env[None]") + + key = self._key(key) + if key: + del self._dict[key] + + def __delitem__(self, key): + self.unset(key) + + def __repr__(self): + return repr(self._dict) + + def set(self, key, value): + if key is None: + raise KeyError("attempting to assign env[None] = %r" % value) + + if value is None: + raise ValueError("attempting to assign env[%s] = None" % key) + + # The `str` conversion is important to avoid triggering errors + # with subprocess if we pass in a unicode value; see commentary + # in the `_key` method. + key = str(key) + value = str(value) + + # The `unset` call is necessary on windows where the keys are + # case insensitive. Since this dict is case sensitive, simply + # assigning the value to the new key is not sufficient to remove + # the old value. The `unset` call knows how to match keys and + # remove any potential duplicates. + self.unset(key) + self._dict[key] = value + + def __setitem__(self, key, value): + self.set(key, value) + + def __iter__(self): + return self._dict.__iter__() + + def __len__(self): + return len(self._dict) + + def keys(self): + return self._dict.keys() + + def values(self): + return self._dict.values() + + def items(self): + return self._dict.items() + + +def add_path_entry(env, name, item, append=True, separator=os.pathsep): + """Cause `item` to be added to the path style env var named + `name` held in the `env` dict. `append` specifies whether + the item is added to the end (the default) or should be + prepended if `name` already exists.""" + val = env.get(name, "") + if len(val) > 0: + val = val.split(separator) + else: + val = [] + if append: + val.append(item) + else: + val.insert(0, item) + env.set(name, separator.join(val)) + + +def add_flag(env, name, flag, append=True): + """Cause `flag` to be added to the CXXFLAGS-style env var named + `name` held in the `env` dict. `append` specifies whether the + flag is added to the end (the default) or should be prepended if + `name` already exists.""" + val = shlex.split(env.get(name, "")) + if append: + val.append(flag) + else: + val.insert(0, flag) + env.set(name, " ".join(val)) + + +_path_search_cache = {} +_not_found = object() + + +def tpx_path(): + return "xplat/testinfra/tpx/ctp.tpx" + + +def path_search(env, exename, defval=None): + """Search for exename in the PATH specified in env. + exename is eg: `ninja` and this function knows to append a .exe + to the end on windows. + Returns the path to the exe if found, or None if either no + PATH is set in env or no executable is found.""" + + path = env.get("PATH", None) + if path is None: + return defval + + # The project hash computation code searches for C++ compilers (g++, clang, etc) + # repeatedly. Cache the result so we don't end up searching for these over and over + # again. + cache_key = (path, exename) + result = _path_search_cache.get(cache_key, _not_found) + if result is _not_found: + result = _perform_path_search(path, exename) + _path_search_cache[cache_key] = result + return result + + +def _perform_path_search(path, exename): + is_win = sys.platform.startswith("win") + if is_win: + exename = "%s.exe" % exename + + for bindir in path.split(os.pathsep): + full_name = os.path.join(bindir, exename) + if os.path.exists(full_name) and os.path.isfile(full_name): + if not is_win and not os.access(full_name, os.X_OK): + continue + return full_name + + return None diff --git a/build/fbcode_builder/getdeps/errors.py b/build/fbcode_builder/getdeps/errors.py new file mode 100644 index 000000000000..3fad1a1ded1b --- /dev/null +++ b/build/fbcode_builder/getdeps/errors.py @@ -0,0 +1,19 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + + +class TransientFailure(Exception): + """Raising this error causes getdeps to return with an error code + that Sandcastle will consider to be a retryable transient + infrastructure error""" + + pass + + +class ManifestNotFound(Exception): + def __init__(self, manifest_name): + super(Exception, self).__init__("Unable to find manifest '%s'" % manifest_name) diff --git a/build/fbcode_builder/getdeps/expr.py b/build/fbcode_builder/getdeps/expr.py new file mode 100644 index 000000000000..6c0485d03fe7 --- /dev/null +++ b/build/fbcode_builder/getdeps/expr.py @@ -0,0 +1,184 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import re +import shlex + + +def parse_expr(expr_text, valid_variables): + """parses the simple criteria expression syntax used in + dependency specifications. + Returns an ExprNode instance that can be evaluated like this: + + ``` + expr = parse_expr("os=windows") + ok = expr.eval({ + "os": "windows" + }) + ``` + + Whitespace is allowed between tokens. The following terms + are recognized: + + KEY = VALUE # Evaluates to True if ctx[KEY] == VALUE + not(EXPR) # Evaluates to True if EXPR evaluates to False + # and vice versa + all(EXPR1, EXPR2, ...) # Evaluates True if all of the supplied + # EXPR's also evaluate True + any(EXPR1, EXPR2, ...) # Evaluates True if any of the supplied + # EXPR's also evaluate True, False if + # none of them evaluated true. + """ + + p = Parser(expr_text, valid_variables) + return p.parse() + + +class ExprNode(object): + def eval(self, ctx): + return False + + +class TrueExpr(ExprNode): + def eval(self, ctx): + return True + + def __str__(self): + return "true" + + +class NotExpr(ExprNode): + def __init__(self, node): + self._node = node + + def eval(self, ctx): + return not self._node.eval(ctx) + + def __str__(self): + return "not(%s)" % self._node + + +class AllExpr(ExprNode): + def __init__(self, nodes): + self._nodes = nodes + + def eval(self, ctx): + for node in self._nodes: + if not node.eval(ctx): + return False + return True + + def __str__(self): + items = [] + for node in self._nodes: + items.append(str(node)) + return "all(%s)" % ",".join(items) + + +class AnyExpr(ExprNode): + def __init__(self, nodes): + self._nodes = nodes + + def eval(self, ctx): + for node in self._nodes: + if node.eval(ctx): + return True + return False + + def __str__(self): + items = [] + for node in self._nodes: + items.append(str(node)) + return "any(%s)" % ",".join(items) + + +class EqualExpr(ExprNode): + def __init__(self, key, value): + self._key = key + self._value = value + + def eval(self, ctx): + return ctx.get(self._key) == self._value + + def __str__(self): + return "%s=%s" % (self._key, self._value) + + +class Parser(object): + def __init__(self, text, valid_variables): + self.text = text + self.lex = shlex.shlex(text) + self.valid_variables = valid_variables + + def parse(self): + expr = self.top() + garbage = self.lex.get_token() + if garbage != "": + raise Exception( + "Unexpected token %s after EqualExpr in %s" % (garbage, self.text) + ) + return expr + + def top(self): + name = self.ident() + op = self.lex.get_token() + + if op == "(": + parsers = { + "not": self.parse_not, + "any": self.parse_any, + "all": self.parse_all, + } + func = parsers.get(name) + if not func: + raise Exception("invalid term %s in %s" % (name, self.text)) + return func() + + if op == "=": + if name not in self.valid_variables: + raise Exception("unknown variable %r in expression" % (name,)) + return EqualExpr(name, self.lex.get_token()) + + raise Exception( + "Unexpected token sequence '%s %s' in %s" % (name, op, self.text) + ) + + def ident(self): + ident = self.lex.get_token() + if not re.match("[a-zA-Z]+", ident): + raise Exception("expected identifier found %s" % ident) + return ident + + def parse_not(self): + node = self.top() + expr = NotExpr(node) + tok = self.lex.get_token() + if tok != ")": + raise Exception("expected ')' found %s" % tok) + return expr + + def parse_any(self): + nodes = [] + while True: + nodes.append(self.top()) + tok = self.lex.get_token() + if tok == ")": + break + if tok != ",": + raise Exception("expected ',' or ')' but found %s" % tok) + return AnyExpr(nodes) + + def parse_all(self): + nodes = [] + while True: + nodes.append(self.top()) + tok = self.lex.get_token() + if tok == ")": + break + if tok != ",": + raise Exception("expected ',' or ')' but found %s" % tok) + return AllExpr(nodes) diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py new file mode 100644 index 000000000000..041549ad726b --- /dev/null +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -0,0 +1,771 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import errno +import hashlib +import os +import re +import shutil +import stat +import subprocess +import sys +import tarfile +import time +import zipfile +from datetime import datetime +from typing import Dict, NamedTuple + +from .copytree import prefetch_dir_if_eden +from .envfuncs import Env +from .errors import TransientFailure +from .platform import is_windows +from .runcmd import run_cmd + + +try: + from urllib import urlretrieve + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + from urllib.request import urlretrieve + + +def file_name_is_cmake_file(file_name): + file_name = file_name.lower() + base = os.path.basename(file_name) + return ( + base.endswith(".cmake") + or base.endswith(".cmake.in") + or base == "cmakelists.txt" + ) + + +class ChangeStatus(object): + """Indicates the nature of changes that happened while updating + the source directory. There are two broad uses: + * When extracting archives for third party software we want to + know that we did something (eg: we either extracted code or + we didn't do anything) + * For 1st party code where we use shipit to transform the code, + we want to know if we changed anything so that we can perform + a build, but we generally want to be a little more nuanced + and be able to distinguish between just changing a source file + and whether we might need to reconfigure the build system. + """ + + def __init__(self, all_changed=False): + """Construct a ChangeStatus object. The default is to create + a status that indicates no changes, but passing all_changed=True + will create one that indicates that everything changed""" + if all_changed: + self.source_files = 1 + self.make_files = 1 + else: + self.source_files = 0 + self.make_files = 0 + + def record_change(self, file_name): + """Used by the shipit fetcher to record changes as it updates + files in the destination. If the file name might be one used + in the cmake build system that we use for 1st party code, then + record that as a "make file" change. We could broaden this + to match any file used by various build systems, but it is + only really useful for our internal cmake stuff at this time. + If the file isn't a build file and is under the `fbcode_builder` + dir then we don't class that as an interesting change that we + might need to rebuild, so we ignore it. + Otherwise we record the file as a source file change.""" + + file_name = file_name.lower() + if file_name_is_cmake_file(file_name): + self.make_files += 1 + elif "/fbcode_builder/cmake" in file_name: + self.source_files += 1 + elif "/fbcode_builder/" not in file_name: + self.source_files += 1 + + def sources_changed(self): + """Returns true if any source files were changed during + an update operation. This will typically be used to decide + that the build system to be run on the source dir in an + incremental mode""" + return self.source_files > 0 + + def build_changed(self): + """Returns true if any build files were changed during + an update operation. This will typically be used to decidfe + that the build system should be reconfigured and re-run + as a full build""" + return self.make_files > 0 + + +class Fetcher(object): + """The Fetcher is responsible for fetching and extracting the + sources for project. The Fetcher instance defines where the + extracted data resides and reports this to the consumer via + its `get_src_dir` method.""" + + def update(self): + """Brings the src dir up to date, ideally minimizing + changes so that a subsequent build doesn't over-build. + Returns a ChangeStatus object that helps the caller to + understand the nature of the changes required during + the update.""" + return ChangeStatus() + + def clean(self): + """Reverts any changes that might have been made to + the src dir""" + pass + + def hash(self): + """Returns a hash that identifies the version of the code in the + working copy. For a git repo this is commit hash for the working + copy. For other Fetchers this should relate to the version of + the code in the src dir. The intent is that if a manifest + changes the version/rev of a project that the hash be different. + Importantly, this should be computable without actually fetching + the code, as we want this to factor into a hash used to download + a pre-built version of the code, without having to first download + and extract its sources (eg: boost on windows is pretty painful). + """ + pass + + def get_src_dir(self): + """Returns the source directory that the project was + extracted into""" + pass + + +class LocalDirFetcher(object): + """This class exists to override the normal fetching behavior, and + use an explicit user-specified directory for the project sources. + + This fetcher cannot update or track changes. It always reports that the + project has changed, forcing it to always be built.""" + + def __init__(self, path): + self.path = os.path.realpath(path) + + def update(self): + return ChangeStatus(all_changed=True) + + def hash(self): + return "0" * 40 + + def get_src_dir(self): + return self.path + + +class SystemPackageFetcher(object): + def __init__(self, build_options, packages): + self.manager = build_options.host_type.get_package_manager() + self.packages = packages.get(self.manager) + if self.packages: + self.installed = None + else: + self.installed = False + + def packages_are_installed(self): + if self.installed is not None: + return self.installed + + if self.manager == "rpm": + result = run_cmd(["rpm", "-q"] + self.packages, allow_fail=True) + self.installed = result == 0 + elif self.manager == "deb": + result = run_cmd(["dpkg", "-s"] + self.packages, allow_fail=True) + self.installed = result == 0 + else: + self.installed = False + + return self.installed + + def update(self): + assert self.installed + return ChangeStatus(all_changed=False) + + def hash(self): + return "0" * 40 + + def get_src_dir(self): + return None + + +class PreinstalledNopFetcher(SystemPackageFetcher): + def __init__(self): + self.installed = True + + +class GitFetcher(Fetcher): + DEFAULT_DEPTH = 1 + + def __init__(self, build_options, manifest, repo_url, rev, depth): + # Extract the host/path portions of the URL and generate a flattened + # directory name. eg: + # github.com/facebook/folly.git -> github.com-facebook-folly.git + url = urlparse(repo_url) + directory = "%s%s" % (url.netloc, url.path) + for s in ["/", "\\", ":"]: + directory = directory.replace(s, "-") + + # Place it in a repos dir in the scratch space + repos_dir = os.path.join(build_options.scratch_dir, "repos") + if not os.path.exists(repos_dir): + os.makedirs(repos_dir) + self.repo_dir = os.path.join(repos_dir, directory) + + if not rev and build_options.project_hashes: + hash_file = os.path.join( + build_options.project_hashes, + re.sub("\\.git$", "-rev.txt", url.path[1:]), + ) + if os.path.exists(hash_file): + with open(hash_file, "r") as f: + data = f.read() + m = re.match("Subproject commit ([a-fA-F0-9]{40})", data) + if not m: + raise Exception("Failed to parse rev from %s" % hash_file) + rev = m.group(1) + print("Using pinned rev %s for %s" % (rev, repo_url)) + + self.rev = rev or "master" + self.origin_repo = repo_url + self.manifest = manifest + self.depth = depth if depth else GitFetcher.DEFAULT_DEPTH + + def _update(self): + current_hash = ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.repo_dir) + .strip() + .decode("utf-8") + ) + target_hash = ( + subprocess.check_output(["git", "rev-parse", self.rev], cwd=self.repo_dir) + .strip() + .decode("utf-8") + ) + if target_hash == current_hash: + # It's up to date, so there are no changes. This doesn't detect eg: + # if origin/master moved and rev='master', but that's ok for our purposes; + # we should be using explicit hashes or eg: a stable branch for the cases + # that we care about, and it isn't unreasonable to require that the user + # explicitly perform a clean build if those have moved. For the most + # part we prefer that folks build using a release tarball from github + # rather than use the git protocol, as it is generally a bit quicker + # to fetch and easier to hash and verify tarball downloads. + return ChangeStatus() + + print("Updating %s -> %s" % (self.repo_dir, self.rev)) + run_cmd(["git", "fetch", "origin", self.rev], cwd=self.repo_dir) + run_cmd(["git", "checkout", self.rev], cwd=self.repo_dir) + run_cmd(["git", "submodule", "update", "--init"], cwd=self.repo_dir) + + return ChangeStatus(True) + + def update(self): + if os.path.exists(self.repo_dir): + return self._update() + self._clone() + return ChangeStatus(True) + + def _clone(self): + print("Cloning %s..." % self.origin_repo) + # The basename/dirname stuff allows us to dance around issues where + # eg: this python process is native win32, but the git.exe is cygwin + # or msys and doesn't like the absolute windows path that we'd otherwise + # pass to it. Careful use of cwd helps avoid headaches with cygpath. + run_cmd( + [ + "git", + "clone", + "--depth=" + str(self.depth), + "--", + self.origin_repo, + os.path.basename(self.repo_dir), + ], + cwd=os.path.dirname(self.repo_dir), + ) + self._update() + + def clean(self): + if os.path.exists(self.repo_dir): + run_cmd(["git", "clean", "-fxd"], cwd=self.repo_dir) + + def hash(self): + return self.rev + + def get_src_dir(self): + return self.repo_dir + + +def does_file_need_update(src_name, src_st, dest_name): + try: + target_st = os.lstat(dest_name) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + return True + + if src_st.st_size != target_st.st_size: + return True + + if stat.S_IFMT(src_st.st_mode) != stat.S_IFMT(target_st.st_mode): + return True + if stat.S_ISLNK(src_st.st_mode): + return os.readlink(src_name) != os.readlink(dest_name) + if not stat.S_ISREG(src_st.st_mode): + return True + + # They might have the same content; compare. + with open(src_name, "rb") as sf, open(dest_name, "rb") as df: + chunk_size = 8192 + while True: + src_data = sf.read(chunk_size) + dest_data = df.read(chunk_size) + if src_data != dest_data: + return True + if len(src_data) < chunk_size: + # EOF + break + return False + + +def copy_if_different(src_name, dest_name): + """Copy src_name -> dest_name, but only touch dest_name + if src_name is different from dest_name, making this a + more build system friendly way to copy.""" + src_st = os.lstat(src_name) + if not does_file_need_update(src_name, src_st, dest_name): + return False + + dest_parent = os.path.dirname(dest_name) + if not os.path.exists(dest_parent): + os.makedirs(dest_parent) + if stat.S_ISLNK(src_st.st_mode): + try: + os.unlink(dest_name) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + target = os.readlink(src_name) + print("Symlinking %s -> %s" % (dest_name, target)) + os.symlink(target, dest_name) + else: + print("Copying %s -> %s" % (src_name, dest_name)) + shutil.copy2(src_name, dest_name) + + return True + + +def list_files_under_dir_newer_than_timestamp(dir_to_scan, ts): + for root, _dirs, files in os.walk(dir_to_scan): + for src_file in files: + full_name = os.path.join(root, src_file) + st = os.lstat(full_name) + if st.st_mtime > ts: + yield full_name + + +class ShipitPathMap(object): + def __init__(self): + self.roots = [] + self.mapping = [] + self.exclusion = [] + + def add_mapping(self, fbsource_dir, target_dir): + """Add a posix path or pattern. We cannot normpath the input + here because that would change the paths from posix to windows + form and break the logic throughout this class.""" + self.roots.append(fbsource_dir) + self.mapping.append((fbsource_dir, target_dir)) + + def add_exclusion(self, pattern): + self.exclusion.append(re.compile(pattern)) + + def _minimize_roots(self): + """compute the de-duplicated set of roots within fbsource. + We take the shortest common directory prefix to make this + determination""" + self.roots.sort(key=len) + minimized = [] + + for r in self.roots: + add_this_entry = True + for existing in minimized: + if r.startswith(existing + "/"): + add_this_entry = False + break + if add_this_entry: + minimized.append(r) + + self.roots = minimized + + def _sort_mapping(self): + self.mapping.sort(reverse=True, key=lambda x: len(x[0])) + + def _map_name(self, norm_name, dest_root): + if norm_name.endswith(".pyc") or norm_name.endswith(".swp"): + # Ignore some incidental garbage while iterating + return None + + for excl in self.exclusion: + if excl.match(norm_name): + return None + + for src_name, dest_name in self.mapping: + if norm_name == src_name or norm_name.startswith(src_name + "/"): + rel_name = os.path.relpath(norm_name, src_name) + # We can have "." as a component of some paths, depending + # on the contents of the shipit transformation section. + # normpath doesn't always remove `.` as the final component + # of the path, which be problematic when we later mkdir + # the dirname of the path that we return. Take care to avoid + # returning a path with a `.` in it. + rel_name = os.path.normpath(rel_name) + if dest_name == ".": + return os.path.normpath(os.path.join(dest_root, rel_name)) + dest_name = os.path.normpath(dest_name) + return os.path.normpath(os.path.join(dest_root, dest_name, rel_name)) + + raise Exception("%s did not match any rules" % norm_name) + + def mirror(self, fbsource_root, dest_root): + self._minimize_roots() + self._sort_mapping() + + change_status = ChangeStatus() + + # Record the full set of files that should be in the tree + full_file_list = set() + + for fbsource_subdir in self.roots: + dir_to_mirror = os.path.join(fbsource_root, fbsource_subdir) + prefetch_dir_if_eden(dir_to_mirror) + if not os.path.exists(dir_to_mirror): + raise Exception( + "%s doesn't exist; check your sparse profile!" % dir_to_mirror + ) + for root, _dirs, files in os.walk(dir_to_mirror): + for src_file in files: + full_name = os.path.join(root, src_file) + rel_name = os.path.relpath(full_name, fbsource_root) + norm_name = rel_name.replace("\\", "/") + + target_name = self._map_name(norm_name, dest_root) + if target_name: + full_file_list.add(target_name) + if copy_if_different(full_name, target_name): + change_status.record_change(target_name) + + # Compare the list of previously shipped files; if a file is + # in the old list but not the new list then it has been + # removed from the source and should be removed from the + # destination. + # Why don't we simply create this list by walking dest_root? + # Some builds currently have to be in-source builds and + # may legitimately need to keep some state in the source tree :-/ + installed_name = os.path.join(dest_root, ".shipit_shipped") + if os.path.exists(installed_name): + with open(installed_name, "rb") as f: + for name in f.read().decode("utf-8").splitlines(): + name = name.strip() + if name not in full_file_list: + print("Remove %s" % name) + os.unlink(name) + change_status.record_change(name) + + with open(installed_name, "wb") as f: + for name in sorted(list(full_file_list)): + f.write(("%s\n" % name).encode("utf-8")) + + return change_status + + +class FbsourceRepoData(NamedTuple): + hash: str + date: str + + +FBSOURCE_REPO_DATA: Dict[str, FbsourceRepoData] = {} + + +def get_fbsource_repo_data(build_options): + """Returns the commit metadata for the fbsource repo. + Since we may have multiple first party projects to + hash, and because we don't mutate the repo, we cache + this hash in a global.""" + cached_data = FBSOURCE_REPO_DATA.get(build_options.fbsource_dir) + if cached_data: + return cached_data + + cmd = ["hg", "log", "-r.", "-T{node}\n{date|hgdate}"] + env = Env() + env.set("HGPLAIN", "1") + log_data = subprocess.check_output( + cmd, cwd=build_options.fbsource_dir, env=dict(env.items()) + ).decode("ascii") + + (hash, datestr) = log_data.split("\n") + + # datestr is like "seconds fractionalseconds" + # We want "20200324.113140" + (unixtime, _fractional) = datestr.split(" ") + date = datetime.fromtimestamp(int(unixtime)).strftime("%Y%m%d.%H%M%S") + cached_data = FbsourceRepoData(hash=hash, date=date) + + FBSOURCE_REPO_DATA[build_options.fbsource_dir] = cached_data + + return cached_data + + +class SimpleShipitTransformerFetcher(Fetcher): + def __init__(self, build_options, manifest): + self.build_options = build_options + self.manifest = manifest + self.repo_dir = os.path.join(build_options.scratch_dir, "shipit", manifest.name) + + def clean(self): + if os.path.exists(self.repo_dir): + shutil.rmtree(self.repo_dir) + + def update(self): + mapping = ShipitPathMap() + for src, dest in self.manifest.get_section_as_ordered_pairs("shipit.pathmap"): + mapping.add_mapping(src, dest) + if self.manifest.shipit_fbcode_builder: + mapping.add_mapping( + "fbcode/opensource/fbcode_builder", "build/fbcode_builder" + ) + for pattern in self.manifest.get_section_as_args("shipit.strip"): + mapping.add_exclusion(pattern) + + return mapping.mirror(self.build_options.fbsource_dir, self.repo_dir) + + def hash(self): + # We return a fixed non-hash string for in-fbsource builds. + # We're relying on the `update` logic to correctly invalidate + # the build in the case that files have changed. + return "fbsource" + + def get_src_dir(self): + return self.repo_dir + + +class ShipitTransformerFetcher(Fetcher): + SHIPIT = "/var/www/scripts/opensource/shipit/run_shipit.php" + + def __init__(self, build_options, project_name): + self.build_options = build_options + self.project_name = project_name + self.repo_dir = os.path.join(build_options.scratch_dir, "shipit", project_name) + + def update(self): + if os.path.exists(self.repo_dir): + return ChangeStatus() + self.run_shipit() + return ChangeStatus(True) + + def clean(self): + if os.path.exists(self.repo_dir): + shutil.rmtree(self.repo_dir) + + @classmethod + def available(cls): + return os.path.exists(cls.SHIPIT) + + def run_shipit(self): + tmp_path = self.repo_dir + ".new" + try: + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) + + # Run shipit + run_cmd( + [ + "php", + ShipitTransformerFetcher.SHIPIT, + "--project=" + self.project_name, + "--create-new-repo", + "--source-repo-dir=" + self.build_options.fbsource_dir, + "--source-branch=.", + "--skip-source-init", + "--skip-source-pull", + "--skip-source-clean", + "--skip-push", + "--skip-reset", + "--destination-use-anonymous-https", + "--create-new-repo-output-path=" + tmp_path, + ] + ) + + # Remove the .git directory from the repository it generated. + # There is no need to commit this. + repo_git_dir = os.path.join(tmp_path, ".git") + shutil.rmtree(repo_git_dir) + os.rename(tmp_path, self.repo_dir) + except Exception: + # Clean up after a failed extraction + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) + self.clean() + raise + + def hash(self): + # We return a fixed non-hash string for in-fbsource builds. + return "fbsource" + + def get_src_dir(self): + return self.repo_dir + + +def download_url_to_file_with_progress(url, file_name): + print("Download %s -> %s ..." % (url, file_name)) + + class Progress(object): + last_report = 0 + + def progress(self, count, block, total): + if total == -1: + total = "(Unknown)" + amount = count * block + + if sys.stdout.isatty(): + sys.stdout.write("\r downloading %s of %s " % (amount, total)) + else: + # When logging to CI logs, avoid spamming the logs and print + # status every few seconds + now = time.time() + if now - self.last_report > 5: + sys.stdout.write(".. %s of %s " % (amount, total)) + self.last_report = now + sys.stdout.flush() + + progress = Progress() + start = time.time() + try: + (_filename, headers) = urlretrieve(url, file_name, reporthook=progress.progress) + except (OSError, IOError) as exc: # noqa: B014 + raise TransientFailure( + "Failed to download %s to %s: %s" % (url, file_name, str(exc)) + ) + + end = time.time() + sys.stdout.write(" [Complete in %f seconds]\n" % (end - start)) + sys.stdout.flush() + print(f"{headers}") + + +class ArchiveFetcher(Fetcher): + def __init__(self, build_options, manifest, url, sha256): + self.manifest = manifest + self.url = url + self.sha256 = sha256 + self.build_options = build_options + + url = urlparse(self.url) + basename = "%s-%s" % (manifest.name, os.path.basename(url.path)) + self.file_name = os.path.join(build_options.scratch_dir, "downloads", basename) + self.src_dir = os.path.join(build_options.scratch_dir, "extracted", basename) + self.hash_file = self.src_dir + ".hash" + + def _verify_hash(self): + h = hashlib.sha256() + with open(self.file_name, "rb") as f: + while True: + block = f.read(8192) + if not block: + break + h.update(block) + digest = h.hexdigest() + if digest != self.sha256: + os.unlink(self.file_name) + raise Exception( + "%s: expected sha256 %s but got %s" % (self.url, self.sha256, digest) + ) + + def _download_dir(self): + """returns the download dir, creating it if it doesn't already exist""" + download_dir = os.path.dirname(self.file_name) + if not os.path.exists(download_dir): + os.makedirs(download_dir) + return download_dir + + def _download(self): + self._download_dir() + download_url_to_file_with_progress(self.url, self.file_name) + self._verify_hash() + + def clean(self): + if os.path.exists(self.src_dir): + shutil.rmtree(self.src_dir) + + def update(self): + try: + with open(self.hash_file, "r") as f: + saved_hash = f.read().strip() + if saved_hash == self.sha256 and os.path.exists(self.src_dir): + # Everything is up to date + return ChangeStatus() + print( + "saved hash %s doesn't match expected hash %s, re-validating" + % (saved_hash, self.sha256) + ) + os.unlink(self.hash_file) + except EnvironmentError: + pass + + # If we got here we know the contents of src_dir are either missing + # or wrong, so blow away whatever happened to be there first. + if os.path.exists(self.src_dir): + shutil.rmtree(self.src_dir) + + # If we already have a file here, make sure it looks legit before + # proceeding: any errors and we just remove it and re-download + if os.path.exists(self.file_name): + try: + self._verify_hash() + except Exception: + if os.path.exists(self.file_name): + os.unlink(self.file_name) + + if not os.path.exists(self.file_name): + self._download() + + if tarfile.is_tarfile(self.file_name): + opener = tarfile.open + elif zipfile.is_zipfile(self.file_name): + opener = zipfile.ZipFile + else: + raise Exception("don't know how to extract %s" % self.file_name) + os.makedirs(self.src_dir) + print("Extract %s -> %s" % (self.file_name, self.src_dir)) + t = opener(self.file_name) + if is_windows(): + # Ensure that we don't fall over when dealing with long paths + # on windows + src = r"\\?\%s" % os.path.normpath(self.src_dir) + else: + src = self.src_dir + # The `str` here is necessary to ensure that we don't pass a unicode + # object down to tarfile.extractall on python2. When extracting + # the boost tarball it makes some assumptions and tries to convert + # a non-ascii path to ascii and throws. + src = str(src) + t.extractall(src) + + with open(self.hash_file, "w") as f: + f.write(self.sha256) + + return ChangeStatus(True) + + def hash(self): + return self.sha256 + + def get_src_dir(self): + return self.src_dir diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py new file mode 100644 index 000000000000..c5f40d2fa8fb --- /dev/null +++ b/build/fbcode_builder/getdeps/load.py @@ -0,0 +1,354 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import base64 +import hashlib +import os + +from . import fetcher +from .envfuncs import path_search +from .errors import ManifestNotFound +from .manifest import ManifestParser + + +class Loader(object): + """The loader allows our tests to patch the load operation""" + + def _list_manifests(self, build_opts): + """Returns a generator that iterates all the available manifests""" + for (path, _, files) in os.walk(build_opts.manifests_dir): + for name in files: + # skip hidden files + if name.startswith("."): + continue + + yield os.path.join(path, name) + + def _load_manifest(self, path): + return ManifestParser(path) + + def load_project(self, build_opts, project_name): + if "/" in project_name or "\\" in project_name: + # Assume this is a path already + return ManifestParser(project_name) + + for manifest in self._list_manifests(build_opts): + if os.path.basename(manifest) == project_name: + return ManifestParser(manifest) + + raise ManifestNotFound(project_name) + + def load_all(self, build_opts): + manifests_by_name = {} + + for manifest in self._list_manifests(build_opts): + m = self._load_manifest(manifest) + + if m.name in manifests_by_name: + raise Exception("found duplicate manifest '%s'" % m.name) + + manifests_by_name[m.name] = m + + return manifests_by_name + + +class ResourceLoader(Loader): + def __init__(self, namespace, manifests_dir): + self.namespace = namespace + self.manifests_dir = manifests_dir + + def _list_manifests(self, _build_opts): + import pkg_resources + + dirs = [self.manifests_dir] + + while dirs: + current = dirs.pop(0) + for name in pkg_resources.resource_listdir(self.namespace, current): + path = "%s/%s" % (current, name) + + if pkg_resources.resource_isdir(self.namespace, path): + dirs.append(path) + else: + yield "%s/%s" % (current, name) + + def _find_manifest(self, project_name): + for name in self._list_manifests(): + if name.endswith("/%s" % project_name): + return name + + raise ManifestNotFound(project_name) + + def _load_manifest(self, path): + import pkg_resources + + contents = pkg_resources.resource_string(self.namespace, path).decode("utf8") + return ManifestParser(file_name=path, fp=contents) + + def load_project(self, build_opts, project_name): + project_name = self._find_manifest(project_name) + return self._load_resource_manifest(project_name) + + +LOADER = Loader() + + +def patch_loader(namespace, manifests_dir="manifests"): + global LOADER + LOADER = ResourceLoader(namespace, manifests_dir) + + +def load_project(build_opts, project_name): + """given the name of a project or a path to a manifest file, + load up the ManifestParser instance for it and return it""" + return LOADER.load_project(build_opts, project_name) + + +def load_all_manifests(build_opts): + return LOADER.load_all(build_opts) + + +class ManifestLoader(object): + """ManifestLoader stores information about project manifest relationships for a + given set of (build options + platform) configuration. + + The ManifestLoader class primarily serves as a location to cache project dependency + relationships and project hash values for this build configuration. + """ + + def __init__(self, build_opts, ctx_gen=None): + self._loader = LOADER + self.build_opts = build_opts + if ctx_gen is None: + self.ctx_gen = self.build_opts.get_context_generator() + else: + self.ctx_gen = ctx_gen + + self.manifests_by_name = {} + self._loaded_all = False + self._project_hashes = {} + self._fetcher_overrides = {} + self._build_dir_overrides = {} + self._install_dir_overrides = {} + self._install_prefix_overrides = {} + + def load_manifest(self, name): + manifest = self.manifests_by_name.get(name) + if manifest is None: + manifest = self._loader.load_project(self.build_opts, name) + self.manifests_by_name[name] = manifest + return manifest + + def load_all_manifests(self): + if not self._loaded_all: + all_manifests_by_name = self._loader.load_all(self.build_opts) + if self.manifests_by_name: + # To help ensure that we only ever have a single manifest object for a + # given project, and that it can't change once we have loaded it, + # only update our mapping for projects that weren't already loaded. + for name, manifest in all_manifests_by_name.items(): + self.manifests_by_name.setdefault(name, manifest) + else: + self.manifests_by_name = all_manifests_by_name + self._loaded_all = True + + return self.manifests_by_name + + def manifests_in_dependency_order(self, manifest=None): + """Compute all dependencies of the specified project. Returns a list of the + dependencies plus the project itself, in topologically sorted order. + + Each entry in the returned list only depends on projects that appear before it + in the list. + + If the input manifest is None, the dependencies for all currently loaded + projects will be computed. i.e., if you call load_all_manifests() followed by + manifests_in_dependency_order() this will return a global dependency ordering of + all projects.""" + # The list of deps that have been fully processed + seen = set() + # The list of deps which have yet to be evaluated. This + # can potentially contain duplicates. + if manifest is None: + deps = list(self.manifests_by_name.values()) + else: + assert manifest.name in self.manifests_by_name + deps = [manifest] + # The list of manifests in dependency order + dep_order = [] + + while len(deps) > 0: + m = deps.pop(0) + if m.name in seen: + continue + + # Consider its deps, if any. + # We sort them for increased determinism; we'll produce + # a correct order even if they aren't sorted, but we prefer + # to produce the same order regardless of how they are listed + # in the project manifest files. + ctx = self.ctx_gen.get_context(m.name) + dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) + builder = m.get("build", "builder", ctx=ctx) + if builder in ("cmake", "python-wheel"): + dep_list.append("cmake") + elif builder == "autoconf" and m.name not in ( + "autoconf", + "libtool", + "automake", + ): + # they need libtool and its deps (automake, autoconf) so add + # those as deps (but obviously not if we're building those + # projects themselves) + dep_list.append("libtool") + + dep_count = 0 + for dep_name in dep_list: + # If we're not sure whether it is done, queue it up + if dep_name not in seen: + dep = self.manifests_by_name.get(dep_name) + if dep is None: + dep = self._loader.load_project(self.build_opts, dep_name) + self.manifests_by_name[dep.name] = dep + + deps.append(dep) + dep_count += 1 + + if dep_count > 0: + # If we queued anything, re-queue this item, as it depends + # those new item(s) and their transitive deps. + deps.append(m) + continue + + # Its deps are done, so we can emit it + seen.add(m.name) + dep_order.append(m) + + return dep_order + + def set_project_src_dir(self, project_name, path): + self._fetcher_overrides[project_name] = fetcher.LocalDirFetcher(path) + + def set_project_build_dir(self, project_name, path): + self._build_dir_overrides[project_name] = path + + def set_project_install_dir(self, project_name, path): + self._install_dir_overrides[project_name] = path + + def set_project_install_prefix(self, project_name, path): + self._install_prefix_overrides[project_name] = path + + def create_fetcher(self, manifest): + override = self._fetcher_overrides.get(manifest.name) + if override is not None: + return override + + ctx = self.ctx_gen.get_context(manifest.name) + return manifest.create_fetcher(self.build_opts, ctx) + + def get_project_hash(self, manifest): + h = self._project_hashes.get(manifest.name) + if h is None: + h = self._compute_project_hash(manifest) + self._project_hashes[manifest.name] = h + return h + + def _compute_project_hash(self, manifest): + """This recursive function computes a hash for a given manifest. + The hash takes into account some environmental factors on the + host machine and includes the hashes of its dependencies. + No caching of the computation is performed, which is theoretically + wasteful but the computation is fast enough that it is not required + to cache across multiple invocations.""" + ctx = self.ctx_gen.get_context(manifest.name) + + hasher = hashlib.sha256() + # Some environmental and configuration things matter + env = {} + env["install_dir"] = self.build_opts.install_dir + env["scratch_dir"] = self.build_opts.scratch_dir + env["vcvars_path"] = self.build_opts.vcvars_path + env["os"] = self.build_opts.host_type.ostype + env["distro"] = self.build_opts.host_type.distro + env["distro_vers"] = self.build_opts.host_type.distrovers + for name in [ + "CXXFLAGS", + "CPPFLAGS", + "LDFLAGS", + "CXX", + "CC", + "GETDEPS_CMAKE_DEFINES", + ]: + env[name] = os.environ.get(name) + for tool in ["cc", "c++", "gcc", "g++", "clang", "clang++"]: + env["tool-%s" % tool] = path_search(os.environ, tool) + for name in manifest.get_section_as_args("depends.environment", ctx): + env[name] = os.environ.get(name) + + fetcher = self.create_fetcher(manifest) + env["fetcher.hash"] = fetcher.hash() + + for name in sorted(env.keys()): + hasher.update(name.encode("utf-8")) + value = env.get(name) + if value is not None: + try: + hasher.update(value.encode("utf-8")) + except AttributeError as exc: + raise AttributeError("name=%r, value=%r: %s" % (name, value, exc)) + + manifest.update_hash(hasher, ctx) + + dep_list = sorted(manifest.get_section_as_dict("dependencies", ctx).keys()) + for dep in dep_list: + dep_manifest = self.load_manifest(dep) + dep_hash = self.get_project_hash(dep_manifest) + hasher.update(dep_hash.encode("utf-8")) + + # Use base64 to represent the hash, rather than the simple hex digest, + # so that the string is shorter. Use the URL-safe encoding so that + # the hash can also be safely used as a filename component. + h = base64.urlsafe_b64encode(hasher.digest()).decode("ascii") + # ... and because cmd.exe is troublesome with `=` signs, nerf those. + # They tend to be padding characters at the end anyway, so we can + # safely discard them. + h = h.replace("=", "") + + return h + + def _get_project_dir_name(self, manifest): + if manifest.is_first_party_project(): + return manifest.name + else: + project_hash = self.get_project_hash(manifest) + return "%s-%s" % (manifest.name, project_hash) + + def get_project_install_dir(self, manifest): + override = self._install_dir_overrides.get(manifest.name) + if override: + return override + + project_dir_name = self._get_project_dir_name(manifest) + return os.path.join(self.build_opts.install_dir, project_dir_name) + + def get_project_build_dir(self, manifest): + override = self._build_dir_overrides.get(manifest.name) + if override: + return override + + project_dir_name = self._get_project_dir_name(manifest) + return os.path.join(self.build_opts.scratch_dir, "build", project_dir_name) + + def get_project_install_prefix(self, manifest): + return self._install_prefix_overrides.get(manifest.name) + + def get_project_install_dir_respecting_install_prefix(self, manifest): + inst_dir = self.get_project_install_dir(manifest) + prefix = self.get_project_install_prefix(manifest) + if prefix: + return inst_dir + prefix + return inst_dir diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py new file mode 100644 index 000000000000..71566d659bb4 --- /dev/null +++ b/build/fbcode_builder/getdeps/manifest.py @@ -0,0 +1,606 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import io +import os + +from .builder import ( + AutoconfBuilder, + Boost, + CargoBuilder, + CMakeBuilder, + BistroBuilder, + Iproute2Builder, + MakeBuilder, + NinjaBootstrap, + NopBuilder, + OpenNSABuilder, + OpenSSLBuilder, + SqliteBuilder, + CMakeBootStrapBuilder, +) +from .expr import parse_expr +from .fetcher import ( + ArchiveFetcher, + GitFetcher, + PreinstalledNopFetcher, + ShipitTransformerFetcher, + SimpleShipitTransformerFetcher, + SystemPackageFetcher, +) +from .py_wheel_builder import PythonWheelBuilder + + +try: + import configparser +except ImportError: + import ConfigParser as configparser + +REQUIRED = "REQUIRED" +OPTIONAL = "OPTIONAL" + +SCHEMA = { + "manifest": { + "optional_section": False, + "fields": { + "name": REQUIRED, + "fbsource_path": OPTIONAL, + "shipit_project": OPTIONAL, + "shipit_fbcode_builder": OPTIONAL, + }, + }, + "dependencies": {"optional_section": True, "allow_values": False}, + "depends.environment": {"optional_section": True}, + "git": { + "optional_section": True, + "fields": {"repo_url": REQUIRED, "rev": OPTIONAL, "depth": OPTIONAL}, + }, + "download": { + "optional_section": True, + "fields": {"url": REQUIRED, "sha256": REQUIRED}, + }, + "build": { + "optional_section": True, + "fields": { + "builder": REQUIRED, + "subdir": OPTIONAL, + "build_in_src_dir": OPTIONAL, + "disable_env_override_pkgconfig": OPTIONAL, + "disable_env_override_path": OPTIONAL, + }, + }, + "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}}, + "cargo": { + "optional_section": True, + "fields": { + "build_doc": OPTIONAL, + "workspace_dir": OPTIONAL, + "manifests_to_build": OPTIONAL, + }, + }, + "cmake.defines": {"optional_section": True}, + "autoconf.args": {"optional_section": True}, + "rpms": {"optional_section": True}, + "debs": {"optional_section": True}, + "preinstalled.env": {"optional_section": True}, + "b2.args": {"optional_section": True}, + "make.build_args": {"optional_section": True}, + "make.install_args": {"optional_section": True}, + "make.test_args": {"optional_section": True}, + "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}}, + "shipit.pathmap": {"optional_section": True}, + "shipit.strip": {"optional_section": True}, + "install.files": {"optional_section": True}, +} + +# These sections are allowed to vary for different platforms +# using the expression syntax to enable/disable sections +ALLOWED_EXPR_SECTIONS = [ + "autoconf.args", + "build", + "cmake.defines", + "dependencies", + "make.build_args", + "make.install_args", + "b2.args", + "download", + "git", + "install.files", +] + + +def parse_conditional_section_name(name, section_def): + expr = name[len(section_def) + 1 :] + return parse_expr(expr, ManifestContext.ALLOWED_VARIABLES) + + +def validate_allowed_fields(file_name, section, config, allowed_fields): + for field in config.options(section): + if not allowed_fields.get(field): + raise Exception( + ("manifest file %s section '%s' contains " "unknown field '%s'") + % (file_name, section, field) + ) + + for field in allowed_fields: + if allowed_fields[field] == REQUIRED and not config.has_option(section, field): + raise Exception( + ("manifest file %s section '%s' is missing " "required field '%s'") + % (file_name, section, field) + ) + + +def validate_allow_values(file_name, section, config): + for field in config.options(section): + value = config.get(section, field) + if value is not None: + raise Exception( + ( + "manifest file %s section '%s' has '%s = %s' but " + "this section doesn't allow specifying values " + "for its entries" + ) + % (file_name, section, field, value) + ) + + +def validate_section(file_name, section, config): + section_def = SCHEMA.get(section) + if not section_def: + for name in ALLOWED_EXPR_SECTIONS: + if section.startswith(name + "."): + # Verify that the conditional parses, but discard it + try: + parse_conditional_section_name(section, name) + except Exception as exc: + raise Exception( + ("manifest file %s section '%s' has invalid " "conditional: %s") + % (file_name, section, str(exc)) + ) + section_def = SCHEMA.get(name) + canonical_section_name = name + break + if not section_def: + raise Exception( + "manifest file %s contains unknown section '%s'" % (file_name, section) + ) + else: + canonical_section_name = section + + allowed_fields = section_def.get("fields") + if allowed_fields: + validate_allowed_fields(file_name, section, config, allowed_fields) + elif not section_def.get("allow_values", True): + validate_allow_values(file_name, section, config) + return canonical_section_name + + +class ManifestParser(object): + def __init__(self, file_name, fp=None): + # allow_no_value enables listing parameters in the + # autoconf.args section one per line + config = configparser.RawConfigParser(allow_no_value=True) + config.optionxform = str # make it case sensitive + + if fp is None: + with open(file_name, "r") as fp: + config.read_file(fp) + elif isinstance(fp, type("")): + # For testing purposes, parse from a string (str + # or unicode) + config.read_file(io.StringIO(fp)) + else: + config.read_file(fp) + + # validate against the schema + seen_sections = set() + + for section in config.sections(): + seen_sections.add(validate_section(file_name, section, config)) + + for section in SCHEMA.keys(): + section_def = SCHEMA[section] + if ( + not section_def.get("optional_section", False) + and section not in seen_sections + ): + raise Exception( + "manifest file %s is missing required section %s" + % (file_name, section) + ) + + self._config = config + self.name = config.get("manifest", "name") + self.fbsource_path = self.get("manifest", "fbsource_path") + self.shipit_project = self.get("manifest", "shipit_project") + self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder") + + if self.name != os.path.basename(file_name): + raise Exception( + "filename of the manifest '%s' does not match the manifest name '%s'" + % (file_name, self.name) + ) + + def get(self, section, key, defval=None, ctx=None): + ctx = ctx or {} + + for s in self._config.sections(): + if s == section: + if self._config.has_option(s, key): + return self._config.get(s, key) + return defval + + if s.startswith(section + "."): + expr = parse_conditional_section_name(s, section) + if not expr.eval(ctx): + continue + + if self._config.has_option(s, key): + return self._config.get(s, key) + + return defval + + def get_section_as_args(self, section, ctx=None): + """Intended for use with the make.[build_args/install_args] and + autoconf.args sections, this method collects the entries and returns an + array of strings. + If the manifest contains conditional sections, ctx is used to + evaluate the condition and merge in the values. + """ + args = [] + ctx = ctx or {} + + for s in self._config.sections(): + if s != section: + if not s.startswith(section + "."): + continue + expr = parse_conditional_section_name(s, section) + if not expr.eval(ctx): + continue + for field in self._config.options(s): + value = self._config.get(s, field) + if value is None: + args.append(field) + else: + args.append("%s=%s" % (field, value)) + return args + + def get_section_as_ordered_pairs(self, section, ctx=None): + """Used for eg: shipit.pathmap which has strong + ordering requirements""" + res = [] + ctx = ctx or {} + + for s in self._config.sections(): + if s != section: + if not s.startswith(section + "."): + continue + expr = parse_conditional_section_name(s, section) + if not expr.eval(ctx): + continue + + for key in self._config.options(s): + value = self._config.get(s, key) + res.append((key, value)) + return res + + def get_section_as_dict(self, section, ctx=None): + d = {} + ctx = ctx or {} + + for s in self._config.sections(): + if s != section: + if not s.startswith(section + "."): + continue + expr = parse_conditional_section_name(s, section) + if not expr.eval(ctx): + continue + for field in self._config.options(s): + value = self._config.get(s, field) + d[field] = value + return d + + def update_hash(self, hasher, ctx): + """Compute a hash over the configuration for the given + context. The goal is for the hash to change if the config + for that context changes, but not if a change is made to + the config only for a different platform than that expressed + by ctx. The hash is intended to be used to help invalidate + a future cache for the third party build products. + The hasher argument is a hash object returned from hashlib.""" + for section in sorted(SCHEMA.keys()): + hasher.update(section.encode("utf-8")) + + # Note: at the time of writing, nothing in the implementation + # relies on keys in any config section being ordered. + # In theory we could have conflicting flags in different + # config sections and later flags override earlier flags. + # For the purposes of computing a hash we're not super + # concerned about this: manifest changes should be rare + # enough and we'd rather that this trigger an invalidation + # than strive for a cache hit at this time. + pairs = self.get_section_as_ordered_pairs(section, ctx) + pairs.sort(key=lambda pair: pair[0]) + for key, value in pairs: + hasher.update(key.encode("utf-8")) + if value is not None: + hasher.update(value.encode("utf-8")) + + def is_first_party_project(self): + """returns true if this is an FB first-party project""" + return self.shipit_project is not None + + def get_required_system_packages(self, ctx): + """Returns dictionary of packager system -> list of packages""" + return { + "rpm": self.get_section_as_args("rpms", ctx), + "deb": self.get_section_as_args("debs", ctx), + } + + def _is_satisfied_by_preinstalled_environment(self, ctx): + envs = self.get_section_as_args("preinstalled.env", ctx) + if not envs: + return False + for key in envs: + val = os.environ.get(key, None) + print(f"Testing ENV[{key}]: {repr(val)}") + if val is None: + return False + if len(val) == 0: + return False + + return True + + def create_fetcher(self, build_options, ctx): + use_real_shipit = ( + ShipitTransformerFetcher.available() and build_options.use_shipit + ) + if ( + not use_real_shipit + and self.fbsource_path + and build_options.fbsource_dir + and self.shipit_project + ): + return SimpleShipitTransformerFetcher(build_options, self) + + if ( + self.fbsource_path + and build_options.fbsource_dir + and self.shipit_project + and ShipitTransformerFetcher.available() + ): + # We can use the code from fbsource + return ShipitTransformerFetcher(build_options, self.shipit_project) + + # Can we satisfy this dep with system packages? + if build_options.allow_system_packages: + if self._is_satisfied_by_preinstalled_environment(ctx): + return PreinstalledNopFetcher() + + packages = self.get_required_system_packages(ctx) + package_fetcher = SystemPackageFetcher(build_options, packages) + if package_fetcher.packages_are_installed(): + return package_fetcher + + repo_url = self.get("git", "repo_url", ctx=ctx) + if repo_url: + rev = self.get("git", "rev") + depth = self.get("git", "depth") + return GitFetcher(build_options, self, repo_url, rev, depth) + + url = self.get("download", "url", ctx=ctx) + if url: + # We need to defer this import until now to avoid triggering + # a cycle when the facebook/__init__.py is loaded. + try: + from getdeps.facebook.lfs import LFSCachingArchiveFetcher + + return LFSCachingArchiveFetcher( + build_options, self, url, self.get("download", "sha256", ctx=ctx) + ) + except ImportError: + # This FB internal module isn't shippped to github, + # so just use its base class + return ArchiveFetcher( + build_options, self, url, self.get("download", "sha256", ctx=ctx) + ) + + raise KeyError( + "project %s has no fetcher configuration matching %s" % (self.name, ctx) + ) + + def create_builder( # noqa:C901 + self, + build_options, + src_dir, + build_dir, + inst_dir, + ctx, + loader, + final_install_prefix=None, + extra_cmake_defines=None, + ): + builder = self.get("build", "builder", ctx=ctx) + if not builder: + raise Exception("project %s has no builder for %r" % (self.name, ctx)) + build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx) + if build_in_src_dir == "true": + # Some scripts don't work when they are configured and build in + # a different directory than source (or when the build directory + # is not a subdir of source). + build_dir = src_dir + subdir = self.get("build", "subdir", None, ctx=ctx) + if subdir is not None: + build_dir = os.path.join(build_dir, subdir) + print("build_dir is %s" % build_dir) # just to quiet lint + + if builder == "make" or builder == "cmakebootstrap": + build_args = self.get_section_as_args("make.build_args", ctx) + install_args = self.get_section_as_args("make.install_args", ctx) + test_args = self.get_section_as_args("make.test_args", ctx) + if builder == "cmakebootstrap": + return CMakeBootStrapBuilder( + build_options, + ctx, + self, + src_dir, + None, + inst_dir, + build_args, + install_args, + test_args, + ) + else: + return MakeBuilder( + build_options, + ctx, + self, + src_dir, + None, + inst_dir, + build_args, + install_args, + test_args, + ) + + if builder == "autoconf": + args = self.get_section_as_args("autoconf.args", ctx) + return AutoconfBuilder( + build_options, ctx, self, src_dir, build_dir, inst_dir, args + ) + + if builder == "boost": + args = self.get_section_as_args("b2.args", ctx) + return Boost(build_options, ctx, self, src_dir, build_dir, inst_dir, args) + + if builder == "bistro": + return BistroBuilder( + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + ) + + if builder == "cmake": + defines = self.get_section_as_dict("cmake.defines", ctx) + return CMakeBuilder( + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + defines, + final_install_prefix, + extra_cmake_defines, + ) + + if builder == "python-wheel": + return PythonWheelBuilder( + build_options, ctx, self, src_dir, build_dir, inst_dir + ) + + if builder == "sqlite": + return SqliteBuilder(build_options, ctx, self, src_dir, build_dir, inst_dir) + + if builder == "ninja_bootstrap": + return NinjaBootstrap( + build_options, ctx, self, build_dir, src_dir, inst_dir + ) + + if builder == "nop": + return NopBuilder(build_options, ctx, self, src_dir, inst_dir) + + if builder == "openssl": + return OpenSSLBuilder( + build_options, ctx, self, build_dir, src_dir, inst_dir + ) + + if builder == "iproute2": + return Iproute2Builder( + build_options, ctx, self, src_dir, build_dir, inst_dir + ) + + if builder == "cargo": + build_doc = self.get("cargo", "build_doc", False, ctx) + workspace_dir = self.get("cargo", "workspace_dir", None, ctx) + manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx) + return CargoBuilder( + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + build_doc, + workspace_dir, + manifests_to_build, + loader, + ) + + if builder == "OpenNSA": + return OpenNSABuilder(build_options, ctx, self, src_dir, inst_dir) + + raise KeyError("project %s has no known builder" % (self.name)) + + +class ManifestContext(object): + """ProjectContext contains a dictionary of values to use when evaluating boolean + expressions in a project manifest. + + This object should be passed as the `ctx` parameter in ManifestParser.get() calls. + """ + + ALLOWED_VARIABLES = {"os", "distro", "distro_vers", "fb", "test"} + + def __init__(self, ctx_dict): + assert set(ctx_dict.keys()) == self.ALLOWED_VARIABLES + self.ctx_dict = ctx_dict + + def get(self, key): + return self.ctx_dict[key] + + def set(self, key, value): + assert key in self.ALLOWED_VARIABLES + self.ctx_dict[key] = value + + def copy(self): + return ManifestContext(dict(self.ctx_dict)) + + def __str__(self): + s = ", ".join( + "%s=%s" % (key, value) for key, value in sorted(self.ctx_dict.items()) + ) + return "{" + s + "}" + + +class ContextGenerator(object): + """ContextGenerator allows creating ManifestContext objects on a per-project basis. + This allows us to evaluate different projects with slightly different contexts. + + For instance, this can be used to only enable tests for some projects.""" + + def __init__(self, default_ctx): + self.default_ctx = ManifestContext(default_ctx) + self.ctx_by_project = {} + + def set_value_for_project(self, project_name, key, value): + project_ctx = self.ctx_by_project.get(project_name) + if project_ctx is None: + project_ctx = self.default_ctx.copy() + self.ctx_by_project[project_name] = project_ctx + project_ctx.set(key, value) + + def set_value_for_all_projects(self, key, value): + self.default_ctx.set(key, value) + for ctx in self.ctx_by_project.values(): + ctx.set(key, value) + + def get_context(self, project_name): + return self.ctx_by_project.get(project_name, self.default_ctx) diff --git a/build/fbcode_builder/getdeps/platform.py b/build/fbcode_builder/getdeps/platform.py new file mode 100644 index 000000000000..fd8382e73220 --- /dev/null +++ b/build/fbcode_builder/getdeps/platform.py @@ -0,0 +1,118 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import platform +import re +import shlex +import sys + + +def is_windows(): + """Returns true if the system we are currently running on + is a Windows system""" + return sys.platform.startswith("win") + + +def get_linux_type(): + try: + with open("/etc/os-release") as f: + data = f.read() + except EnvironmentError: + return (None, None) + + os_vars = {} + for line in data.splitlines(): + parts = line.split("=", 1) + if len(parts) != 2: + continue + key = parts[0].strip() + value_parts = shlex.split(parts[1].strip()) + if not value_parts: + value = "" + else: + value = value_parts[0] + os_vars[key] = value + + name = os_vars.get("NAME") + if name: + name = name.lower() + name = re.sub("linux", "", name) + name = name.strip() + + version_id = os_vars.get("VERSION_ID") + if version_id: + version_id = version_id.lower() + + return "linux", name, version_id + + +class HostType(object): + def __init__(self, ostype=None, distro=None, distrovers=None): + if ostype is None: + distro = None + distrovers = None + if sys.platform.startswith("linux"): + ostype, distro, distrovers = get_linux_type() + elif sys.platform.startswith("darwin"): + ostype = "darwin" + elif is_windows(): + ostype = "windows" + distrovers = str(sys.getwindowsversion().major) + else: + ostype = sys.platform + + # The operating system type + self.ostype = ostype + # The distribution, if applicable + self.distro = distro + # The OS/distro version if known + self.distrovers = distrovers + machine = platform.machine().lower() + if "arm" in machine or "aarch" in machine: + self.isarm = True + else: + self.isarm = False + + def is_windows(self): + return self.ostype == "windows" + + def is_arm(self): + return self.isarm + + def is_darwin(self): + return self.ostype == "darwin" + + def is_linux(self): + return self.ostype == "linux" + + def as_tuple_string(self): + return "%s-%s-%s" % ( + self.ostype, + self.distro or "none", + self.distrovers or "none", + ) + + def get_package_manager(self): + if not self.is_linux(): + return None + if self.distro in ("fedora", "centos"): + return "rpm" + if self.distro in ("debian", "ubuntu"): + return "deb" + return None + + @staticmethod + def from_tuple_string(s): + ostype, distro, distrovers = s.split("-") + return HostType(ostype=ostype, distro=distro, distrovers=distrovers) + + def __eq__(self, b): + return ( + self.ostype == b.ostype + and self.distro == b.distro + and self.distrovers == b.distrovers + ) diff --git a/build/fbcode_builder/getdeps/py_wheel_builder.py b/build/fbcode_builder/getdeps/py_wheel_builder.py new file mode 100644 index 000000000000..82ad8b8071b1 --- /dev/null +++ b/build/fbcode_builder/getdeps/py_wheel_builder.py @@ -0,0 +1,289 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import codecs +import collections +import email +import os +import re +import stat + +from .builder import BuilderBase, CMakeBuilder + + +WheelNameInfo = collections.namedtuple( + "WheelNameInfo", ("distribution", "version", "build", "python", "abi", "platform") +) + +CMAKE_HEADER = """ +cmake_minimum_required(VERSION 3.8) + +project("{manifest_name}" LANGUAGES C) + +set(CMAKE_MODULE_PATH + "{cmake_dir}" + ${{CMAKE_MODULE_PATH}} +) +include(FBPythonBinary) + +set(CMAKE_INSTALL_DIR lib/cmake/{manifest_name} CACHE STRING + "The subdirectory where CMake package config files should be installed") +""" + +CMAKE_FOOTER = """ +install_fb_python_library({lib_name} EXPORT all) +install( + EXPORT all + FILE {manifest_name}-targets.cmake + NAMESPACE {namespace}:: + DESTINATION ${{CMAKE_INSTALL_DIR}} +) + +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${{CMAKE_BINARY_DIR}}/{manifest_name}-config.cmake.in + {manifest_name}-config.cmake + INSTALL_DESTINATION ${{CMAKE_INSTALL_DIR}} + PATH_VARS + CMAKE_INSTALL_DIR +) +install( + FILES ${{CMAKE_CURRENT_BINARY_DIR}}/{manifest_name}-config.cmake + DESTINATION ${{CMAKE_INSTALL_DIR}} +) +""" + +CMAKE_CONFIG_FILE = """ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +set_and_check({upper_name}_CMAKE_DIR "@PACKAGE_CMAKE_INSTALL_DIR@") + +if (NOT TARGET {namespace}::{lib_name}) + include("${{{upper_name}_CMAKE_DIR}}/{manifest_name}-targets.cmake") +endif() + +set({upper_name}_LIBRARIES {namespace}::{lib_name}) + +{find_dependency_lines} + +if (NOT {manifest_name}_FIND_QUIETLY) + message(STATUS "Found {manifest_name}: ${{PACKAGE_PREFIX_DIR}}") +endif() +""" + + +# Note: for now we are manually manipulating the wheel packet contents. +# The wheel format is documented here: +# https://www.python.org/dev/peps/pep-0491/#file-format +# +# We currently aren't particularly smart about correctly handling the full wheel +# functionality, but this is good enough to handle simple pure-python wheels, +# which is the main thing we care about right now. +# +# We could potentially use pip to install the wheel to a temporary location and +# then copy its "installed" files, but this has its own set of complications. +# This would require pip to already be installed and available, and we would +# need to correctly find the right version of pip or pip3 to use. +# If we did ever want to go down that path, we would probably want to use +# something like the following pip3 command: +# pip3 --isolated install --no-cache-dir --no-index --system \ +# --target +class PythonWheelBuilder(BuilderBase): + """This Builder can take Python wheel archives and install them as python libraries + that can be used by add_fb_python_library()/add_fb_python_executable() CMake rules. + """ + + def _build(self, install_dirs, reconfigure): + # type: (List[str], bool) -> None + + # When we are invoked, self.src_dir contains the unpacked wheel contents. + # + # Since a wheel file is just a zip file, the Fetcher code recognizes it as such + # and goes ahead and unpacks it. (We could disable that Fetcher behavior in the + # future if we ever wanted to, say if we wanted to call pip here.) + wheel_name = self._parse_wheel_name() + name_version_prefix = "-".join((wheel_name.distribution, wheel_name.version)) + dist_info_name = name_version_prefix + ".dist-info" + data_dir_name = name_version_prefix + ".data" + self.dist_info_dir = os.path.join(self.src_dir, dist_info_name) + wheel_metadata = self._read_wheel_metadata(wheel_name) + + # Check that we can understand the wheel version. + # We don't really care about wheel_metadata["Root-Is-Purelib"] since + # we are generating our own standalone python archives rather than installing + # into site-packages. + version = wheel_metadata["Wheel-Version"] + if not version.startswith("1."): + raise Exception("unsupported wheel version %s" % (version,)) + + # Add a find_dependency() call for each of our dependencies. + # The dependencies are also listed in the wheel METADATA file, but it is simpler + # to pull this directly from the getdeps manifest. + dep_list = sorted( + self.manifest.get_section_as_dict("dependencies", self.ctx).keys() + ) + find_dependency_lines = ["find_dependency({})".format(dep) for dep in dep_list] + + getdeps_cmake_dir = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "CMake" + ) + self.template_format_dict = { + # Note that CMake files always uses forward slash separators in path names, + # even on Windows. Therefore replace path separators here. + "cmake_dir": _to_cmake_path(getdeps_cmake_dir), + "lib_name": self.manifest.name, + "manifest_name": self.manifest.name, + "namespace": self.manifest.name, + "upper_name": self.manifest.name.upper().replace("-", "_"), + "find_dependency_lines": "\n".join(find_dependency_lines), + } + + # Find sources from the root directory + path_mapping = {} + for entry in os.listdir(self.src_dir): + if entry in (dist_info_name, data_dir_name): + continue + self._add_sources(path_mapping, os.path.join(self.src_dir, entry), entry) + + # Files under the .data directory also need to be installed in the correct + # locations + if os.path.exists(data_dir_name): + # TODO: process the subdirectories of data_dir_name + # This isn't implemented yet since for now we have only needed dependencies + # on some simple pure Python wheels, so I haven't tested against wheels with + # additional files in the .data directory. + raise Exception( + "handling of the subdirectories inside %s is not implemented yet" + % data_dir_name + ) + + # Emit CMake files + self._write_cmakelists(path_mapping, dep_list) + self._write_cmake_config_template() + + # Run the build + self._run_cmake_build(install_dirs, reconfigure) + + def _run_cmake_build(self, install_dirs, reconfigure): + # type: (List[str], bool) -> None + + cmake_builder = CMakeBuilder( + build_opts=self.build_opts, + ctx=self.ctx, + manifest=self.manifest, + # Note that we intentionally supply src_dir=build_dir, + # since we wrote out our generated CMakeLists.txt in the build directory + src_dir=self.build_dir, + build_dir=self.build_dir, + inst_dir=self.inst_dir, + defines={}, + final_install_prefix=None, + ) + cmake_builder.build(install_dirs=install_dirs, reconfigure=reconfigure) + + def _write_cmakelists(self, path_mapping, dependencies): + # type: (List[str]) -> None + + cmake_path = os.path.join(self.build_dir, "CMakeLists.txt") + with open(cmake_path, "w") as f: + f.write(CMAKE_HEADER.format(**self.template_format_dict)) + for dep in dependencies: + f.write("find_package({0} REQUIRED)\n".format(dep)) + + f.write( + "add_fb_python_library({lib_name}\n".format(**self.template_format_dict) + ) + f.write(' BASE_DIR "%s"\n' % _to_cmake_path(self.src_dir)) + f.write(" SOURCES\n") + for src_path, install_path in path_mapping.items(): + f.write( + ' "%s=%s"\n' + % (_to_cmake_path(src_path), _to_cmake_path(install_path)) + ) + if dependencies: + f.write(" DEPENDS\n") + for dep in dependencies: + f.write(' "{0}::{0}"\n'.format(dep)) + f.write(")\n") + + f.write(CMAKE_FOOTER.format(**self.template_format_dict)) + + def _write_cmake_config_template(self): + config_path_name = self.manifest.name + "-config.cmake.in" + output_path = os.path.join(self.build_dir, config_path_name) + + with open(output_path, "w") as f: + f.write(CMAKE_CONFIG_FILE.format(**self.template_format_dict)) + + def _add_sources(self, path_mapping, src_path, install_path): + # type: (List[str], str, str) -> None + + s = os.lstat(src_path) + if not stat.S_ISDIR(s.st_mode): + path_mapping[src_path] = install_path + return + + for entry in os.listdir(src_path): + self._add_sources( + path_mapping, + os.path.join(src_path, entry), + os.path.join(install_path, entry), + ) + + def _parse_wheel_name(self): + # type: () -> WheelNameInfo + + # The ArchiveFetcher prepends "manifest_name-", so strip that off first. + wheel_name = os.path.basename(self.src_dir) + prefix = self.manifest.name + "-" + if not wheel_name.startswith(prefix): + raise Exception( + "expected wheel source directory to be of the form %s-NAME.whl" + % (prefix,) + ) + wheel_name = wheel_name[len(prefix) :] + + wheel_name_re = re.compile( + r"(?P[^-]+)" + r"-(?P\d+[^-]*)" + r"(-(?P\d+[^-]*))?" + r"-(?P\w+\d+(\.\w+\d+)*)" + r"-(?P\w+)" + r"-(?P\w+(\.\w+)*)" + r"\.whl" + ) + match = wheel_name_re.match(wheel_name) + if not match: + raise Exception( + "bad python wheel name %s: expected to have the form " + "DISTRIBUTION-VERSION-[-BUILD]-PYTAG-ABI-PLATFORM" + ) + + return WheelNameInfo( + distribution=match.group("distribution"), + version=match.group("version"), + build=match.group("build"), + python=match.group("python"), + abi=match.group("abi"), + platform=match.group("platform"), + ) + + def _read_wheel_metadata(self, wheel_name): + metadata_path = os.path.join(self.dist_info_dir, "WHEEL") + with codecs.open(metadata_path, "r", encoding="utf-8") as f: + return email.message_from_file(f) + + +def _to_cmake_path(path): + # CMake always uses forward slashes to separate paths in CMakeLists.txt files, + # even on Windows. It treats backslashes as character escapes, so using + # backslashes in the path will cause problems. Therefore replace all path + # separators with forward slashes to make sure the paths are correct on Windows. + # e.g. "C:\foo\bar.txt" becomes "C:/foo/bar.txt" + return path.replace(os.path.sep, "/") diff --git a/build/fbcode_builder/getdeps/runcmd.py b/build/fbcode_builder/getdeps/runcmd.py new file mode 100644 index 000000000000..44e7994aac75 --- /dev/null +++ b/build/fbcode_builder/getdeps/runcmd.py @@ -0,0 +1,169 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import select +import subprocess +import sys + +from .envfuncs import Env +from .platform import is_windows + + +try: + from shlex import quote as shellquote +except ImportError: + from pipes import quote as shellquote + + +class RunCommandError(Exception): + pass + + +def _print_env_diff(env, log_fn): + current_keys = set(os.environ.keys()) + wanted_env = set(env.keys()) + + unset_keys = current_keys.difference(wanted_env) + for k in sorted(unset_keys): + log_fn("+ unset %s\n" % k) + + added_keys = wanted_env.difference(current_keys) + for k in wanted_env.intersection(current_keys): + if os.environ[k] != env[k]: + added_keys.add(k) + + for k in sorted(added_keys): + if ("PATH" in k) and (os.pathsep in env[k]): + log_fn("+ %s=\\\n" % k) + for elem in env[k].split(os.pathsep): + log_fn("+ %s%s\\\n" % (shellquote(elem), os.pathsep)) + else: + log_fn("+ %s=%s \\\n" % (k, shellquote(env[k]))) + + +def run_cmd(cmd, env=None, cwd=None, allow_fail=False, log_file=None): + def log_to_stdout(msg): + sys.stdout.buffer.write(msg.encode(errors="surrogateescape")) + + if log_file is not None: + with open(log_file, "a", encoding="utf-8", errors="surrogateescape") as log: + + def log_function(msg): + log.write(msg) + log_to_stdout(msg) + + return _run_cmd( + cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_function + ) + else: + return _run_cmd( + cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_to_stdout + ) + + +def _run_cmd(cmd, env, cwd, allow_fail, log_fn): + log_fn("---\n") + try: + cmd_str = " \\\n+ ".join(shellquote(arg) for arg in cmd) + except TypeError: + # eg: one of the elements is None + raise RunCommandError("problem quoting cmd: %r" % cmd) + + if env: + assert isinstance(env, Env) + _print_env_diff(env, log_fn) + + # Convert from our Env type to a regular dict. + # This is needed because python3 looks up b'PATH' and 'PATH' + # and emits an error if both are present. In our Env type + # we'll return the same value for both requests, but we don't + # have duplicate potentially conflicting values which is the + # spirit of the check. + env = dict(env.items()) + + if cwd: + log_fn("+ cd %s && \\\n" % shellquote(cwd)) + # Our long path escape sequence may confuse cmd.exe, so if the cwd + # is short enough, strip that off. + if is_windows() and (len(cwd) < 250) and cwd.startswith("\\\\?\\"): + cwd = cwd[4:] + + log_fn("+ %s\n" % cmd_str) + + isinteractive = os.isatty(sys.stdout.fileno()) + if isinteractive: + stdout = None + sys.stdout.buffer.flush() + else: + stdout = subprocess.PIPE + + try: + p = subprocess.Popen( + cmd, env=env, cwd=cwd, stdout=stdout, stderr=subprocess.STDOUT + ) + except (TypeError, ValueError, OSError) as exc: + log_fn("error running `%s`: %s" % (cmd_str, exc)) + raise RunCommandError( + "%s while running `%s` with env=%r\nos.environ=%r" + % (str(exc), cmd_str, env, os.environ) + ) + + if not isinteractive: + _pipe_output(p, log_fn) + + p.wait() + if p.returncode != 0 and not allow_fail: + raise subprocess.CalledProcessError(p.returncode, cmd) + + return p.returncode + + +if hasattr(select, "poll"): + + def _pipe_output(p, log_fn): + """Read output from p.stdout and call log_fn() with each chunk of data as it + becomes available.""" + # Perform non-blocking reads + import fcntl + + fcntl.fcntl(p.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) + poll = select.poll() + poll.register(p.stdout.fileno(), select.POLLIN) + + buffer_size = 4096 + while True: + poll.poll() + data = p.stdout.read(buffer_size) + if not data: + break + # log_fn() accepts arguments as str (binary in Python 2, unicode in + # Python 3). In Python 3 the subprocess output will be plain bytes, + # and need to be decoded. + if not isinstance(data, str): + data = data.decode("utf-8", errors="surrogateescape") + log_fn(data) + + +else: + + def _pipe_output(p, log_fn): + """Read output from p.stdout and call log_fn() with each chunk of data as it + becomes available.""" + # Perform blocking reads. Use a smaller buffer size to avoid blocking + # for very long when data is available. + buffer_size = 64 + while True: + data = p.stdout.read(buffer_size) + if not data: + break + # log_fn() accepts arguments as str (binary in Python 2, unicode in + # Python 3). In Python 3 the subprocess output will be plain bytes, + # and need to be decoded. + if not isinstance(data, str): + data = data.decode("utf-8", errors="surrogateescape") + log_fn(data) diff --git a/build/fbcode_builder/getdeps/subcmd.py b/build/fbcode_builder/getdeps/subcmd.py new file mode 100644 index 000000000000..95f9a07ca02c --- /dev/null +++ b/build/fbcode_builder/getdeps/subcmd.py @@ -0,0 +1,58 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + + +class SubCmd(object): + NAME = None + HELP = None + + def run(self, args): + """perform the command""" + return 0 + + def setup_parser(self, parser): + # Subclasses should override setup_parser() if they have any + # command line options or arguments. + pass + + +CmdTable = [] + + +def add_subcommands(parser, common_args, cmd_table=CmdTable): + """Register parsers for the defined commands with the provided parser""" + for cls in cmd_table: + command = cls() + command_parser = parser.add_parser( + command.NAME, help=command.HELP, parents=[common_args] + ) + command.setup_parser(command_parser) + command_parser.set_defaults(func=command.run) + + +def cmd(name, help=None, cmd_table=CmdTable): + """ + @cmd() is a decorator that can be used to help define Subcmd instances + + Example usage: + + @subcmd('list', 'Show the result list') + class ListCmd(Subcmd): + def run(self, args): + # Perform the command actions here... + pass + """ + + def wrapper(cls): + class SubclassedCmd(cls): + NAME = name + HELP = help + + cmd_table.append(SubclassedCmd) + return SubclassedCmd + + return wrapper diff --git a/build/fbcode_builder/getdeps/test/expr_test.py b/build/fbcode_builder/getdeps/test/expr_test.py new file mode 100644 index 000000000000..59d66a9431bd --- /dev/null +++ b/build/fbcode_builder/getdeps/test/expr_test.py @@ -0,0 +1,49 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import unittest + +from ..expr import parse_expr + + +class ExprTest(unittest.TestCase): + def test_equal(self): + valid_variables = {"foo", "some_var", "another_var"} + e = parse_expr("foo=bar", valid_variables) + self.assertTrue(e.eval({"foo": "bar"})) + self.assertFalse(e.eval({"foo": "not-bar"})) + self.assertFalse(e.eval({"not-foo": "bar"})) + + def test_not_equal(self): + valid_variables = {"foo"} + e = parse_expr("not(foo=bar)", valid_variables) + self.assertFalse(e.eval({"foo": "bar"})) + self.assertTrue(e.eval({"foo": "not-bar"})) + + def test_bad_not(self): + valid_variables = {"foo"} + with self.assertRaises(Exception): + parse_expr("foo=not(bar)", valid_variables) + + def test_bad_variable(self): + valid_variables = {"bar"} + with self.assertRaises(Exception): + parse_expr("foo=bar", valid_variables) + + def test_all(self): + valid_variables = {"foo", "baz"} + e = parse_expr("all(foo = bar, baz = qux)", valid_variables) + self.assertTrue(e.eval({"foo": "bar", "baz": "qux"})) + self.assertFalse(e.eval({"foo": "bar", "baz": "nope"})) + self.assertFalse(e.eval({"foo": "nope", "baz": "nope"})) + + def test_any(self): + valid_variables = {"foo", "baz"} + e = parse_expr("any(foo = bar, baz = qux)", valid_variables) + self.assertTrue(e.eval({"foo": "bar", "baz": "qux"})) + self.assertTrue(e.eval({"foo": "bar", "baz": "nope"})) + self.assertFalse(e.eval({"foo": "nope", "baz": "nope"})) diff --git a/build/fbcode_builder/getdeps/test/fixtures/duplicate/foo b/build/fbcode_builder/getdeps/test/fixtures/duplicate/foo new file mode 100644 index 000000000000..a0384ee3b33f --- /dev/null +++ b/build/fbcode_builder/getdeps/test/fixtures/duplicate/foo @@ -0,0 +1,2 @@ +[manifest] +name = foo diff --git a/build/fbcode_builder/getdeps/test/fixtures/duplicate/subdir/foo b/build/fbcode_builder/getdeps/test/fixtures/duplicate/subdir/foo new file mode 100644 index 000000000000..a0384ee3b33f --- /dev/null +++ b/build/fbcode_builder/getdeps/test/fixtures/duplicate/subdir/foo @@ -0,0 +1,2 @@ +[manifest] +name = foo diff --git a/build/fbcode_builder/getdeps/test/manifest_test.py b/build/fbcode_builder/getdeps/test/manifest_test.py new file mode 100644 index 000000000000..8be9896d840c --- /dev/null +++ b/build/fbcode_builder/getdeps/test/manifest_test.py @@ -0,0 +1,233 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import sys +import unittest + +from ..load import load_all_manifests, patch_loader +from ..manifest import ManifestParser + + +class ManifestTest(unittest.TestCase): + def test_missing_section(self): + with self.assertRaisesRegex( + Exception, "manifest file test is missing required section manifest" + ): + ManifestParser("test", "") + + def test_missing_name(self): + with self.assertRaisesRegex( + Exception, + "manifest file test section 'manifest' is missing required field 'name'", + ): + ManifestParser( + "test", + """ +[manifest] +""", + ) + + def test_minimal(self): + p = ManifestParser( + "test", + """ +[manifest] +name = test +""", + ) + self.assertEqual(p.name, "test") + self.assertEqual(p.fbsource_path, None) + + def test_minimal_with_fbsource_path(self): + p = ManifestParser( + "test", + """ +[manifest] +name = test +fbsource_path = fbcode/wat +""", + ) + self.assertEqual(p.name, "test") + self.assertEqual(p.fbsource_path, "fbcode/wat") + + def test_unknown_field(self): + with self.assertRaisesRegex( + Exception, + ( + "manifest file test section 'manifest' contains " + "unknown field 'invalid.field'" + ), + ): + ManifestParser( + "test", + """ +[manifest] +name = test +invalid.field = woot +""", + ) + + def test_invalid_section_name(self): + with self.assertRaisesRegex( + Exception, "manifest file test contains unknown section 'invalid.section'" + ): + ManifestParser( + "test", + """ +[manifest] +name = test + +[invalid.section] +foo = bar +""", + ) + + def test_value_in_dependencies_section(self): + with self.assertRaisesRegex( + Exception, + ( + "manifest file test section 'dependencies' has " + "'foo = bar' but this section doesn't allow " + "specifying values for its entries" + ), + ): + ManifestParser( + "test", + """ +[manifest] +name = test + +[dependencies] +foo = bar +""", + ) + + def test_invalid_conditional_section_name(self): + with self.assertRaisesRegex( + Exception, + ( + "manifest file test section 'dependencies.=' " + "has invalid conditional: expected " + "identifier found =" + ), + ): + ManifestParser( + "test", + """ +[manifest] +name = test + +[dependencies.=] +""", + ) + + def test_section_as_args(self): + p = ManifestParser( + "test", + """ +[manifest] +name = test + +[dependencies] +a +b +c + +[dependencies.test=on] +foo +""", + ) + self.assertEqual(p.get_section_as_args("dependencies"), ["a", "b", "c"]) + self.assertEqual( + p.get_section_as_args("dependencies", {"test": "off"}), ["a", "b", "c"] + ) + self.assertEqual( + p.get_section_as_args("dependencies", {"test": "on"}), + ["a", "b", "c", "foo"], + ) + + p2 = ManifestParser( + "test", + """ +[manifest] +name = test + +[autoconf.args] +--prefix=/foo +--with-woot +""", + ) + self.assertEqual( + p2.get_section_as_args("autoconf.args"), ["--prefix=/foo", "--with-woot"] + ) + + def test_section_as_dict(self): + p = ManifestParser( + "test", + """ +[manifest] +name = test + +[cmake.defines] +foo = bar + +[cmake.defines.test=on] +foo = baz +""", + ) + self.assertEqual(p.get_section_as_dict("cmake.defines"), {"foo": "bar"}) + self.assertEqual( + p.get_section_as_dict("cmake.defines", {"test": "on"}), {"foo": "baz"} + ) + + p2 = ManifestParser( + "test", + """ +[manifest] +name = test + +[cmake.defines.test=on] +foo = baz + +[cmake.defines] +foo = bar +""", + ) + self.assertEqual( + p2.get_section_as_dict("cmake.defines", {"test": "on"}), + {"foo": "bar"}, + msg="sections cascade in the order they appear in the manifest", + ) + + def test_parse_common_manifests(self): + patch_loader(__name__) + manifests = load_all_manifests(None) + self.assertNotEqual(0, len(manifests), msg="parsed some number of manifests") + + def test_mismatch_name(self): + with self.assertRaisesRegex( + Exception, + "filename of the manifest 'foo' does not match the manifest name 'bar'", + ): + ManifestParser( + "foo", + """ +[manifest] +name = bar +""", + ) + + def test_duplicate_manifest(self): + patch_loader(__name__, "fixtures/duplicate") + + with self.assertRaisesRegex(Exception, "found duplicate manifest 'foo'"): + load_all_manifests(None) + + if sys.version_info < (3, 2): + + def assertRaisesRegex(self, *args, **kwargs): + return self.assertRaisesRegexp(*args, **kwargs) diff --git a/build/fbcode_builder/getdeps/test/platform_test.py b/build/fbcode_builder/getdeps/test/platform_test.py new file mode 100644 index 000000000000..311e9c76cdf2 --- /dev/null +++ b/build/fbcode_builder/getdeps/test/platform_test.py @@ -0,0 +1,40 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import unittest + +from ..platform import HostType + + +class PlatformTest(unittest.TestCase): + def test_create(self): + p = HostType() + self.assertNotEqual(p.ostype, None, msg="probed and returned something") + + tuple_string = p.as_tuple_string() + round_trip = HostType.from_tuple_string(tuple_string) + self.assertEqual(round_trip, p) + + def test_rendering_of_none(self): + p = HostType(ostype="foo") + self.assertEqual(p.as_tuple_string(), "foo-none-none") + + def test_is_methods(self): + p = HostType(ostype="windows") + self.assertTrue(p.is_windows()) + self.assertFalse(p.is_darwin()) + self.assertFalse(p.is_linux()) + + p = HostType(ostype="darwin") + self.assertFalse(p.is_windows()) + self.assertTrue(p.is_darwin()) + self.assertFalse(p.is_linux()) + + p = HostType(ostype="linux") + self.assertFalse(p.is_windows()) + self.assertFalse(p.is_darwin()) + self.assertTrue(p.is_linux()) diff --git a/build/fbcode_builder/getdeps/test/scratch_test.py b/build/fbcode_builder/getdeps/test/scratch_test.py new file mode 100644 index 000000000000..1f43c5951d55 --- /dev/null +++ b/build/fbcode_builder/getdeps/test/scratch_test.py @@ -0,0 +1,80 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import absolute_import, division, print_function + +import unittest + +from ..buildopts import find_existing_win32_subst_for_path + + +class Win32SubstTest(unittest.TestCase): + def test_no_existing_subst(self): + self.assertIsNone( + find_existing_win32_subst_for_path( + r"C:\users\alice\appdata\local\temp\fbcode_builder_getdeps", + subst_mapping={}, + ) + ) + self.assertIsNone( + find_existing_win32_subst_for_path( + r"C:\users\alice\appdata\local\temp\fbcode_builder_getdeps", + subst_mapping={"X:\\": r"C:\users\alice\appdata\local\temp\other"}, + ) + ) + + def test_exact_match_returns_drive_path(self): + self.assertEqual( + find_existing_win32_subst_for_path( + r"C:\temp\fbcode_builder_getdeps", + subst_mapping={"X:\\": r"C:\temp\fbcode_builder_getdeps"}, + ), + "X:\\", + ) + self.assertEqual( + find_existing_win32_subst_for_path( + r"C:/temp/fbcode_builder_getdeps", + subst_mapping={"X:\\": r"C:/temp/fbcode_builder_getdeps"}, + ), + "X:\\", + ) + + def test_multiple_exact_matches_returns_arbitrary_drive_path(self): + self.assertIn( + find_existing_win32_subst_for_path( + r"C:\temp\fbcode_builder_getdeps", + subst_mapping={ + "X:\\": r"C:\temp\fbcode_builder_getdeps", + "Y:\\": r"C:\temp\fbcode_builder_getdeps", + "Z:\\": r"C:\temp\fbcode_builder_getdeps", + }, + ), + ("X:\\", "Y:\\", "Z:\\"), + ) + + def test_drive_letter_is_case_insensitive(self): + self.assertEqual( + find_existing_win32_subst_for_path( + r"C:\temp\fbcode_builder_getdeps", + subst_mapping={"X:\\": r"c:\temp\fbcode_builder_getdeps"}, + ), + "X:\\", + ) + + def test_path_components_are_case_insensitive(self): + self.assertEqual( + find_existing_win32_subst_for_path( + r"C:\TEMP\FBCODE_builder_getdeps", + subst_mapping={"X:\\": r"C:\temp\fbcode_builder_getdeps"}, + ), + "X:\\", + ) + self.assertEqual( + find_existing_win32_subst_for_path( + r"C:\temp\fbcode_builder_getdeps", + subst_mapping={"X:\\": r"C:\TEMP\FBCODE_builder_getdeps"}, + ), + "X:\\", + ) diff --git a/build/fbcode_builder/make_docker_context.py b/build/fbcode_builder/make_docker_context.py new file mode 100755 index 000000000000..d4b0f0a89938 --- /dev/null +++ b/build/fbcode_builder/make_docker_context.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +""" +Reads `fbcode_builder_config.py` from the current directory, and prepares a +Docker context directory to build this project. Prints to stdout the path +to the context directory. + +Try `.../make_docker_context.py --help` from a project's `build/` directory. + +By default, the Docker context directory will be in /tmp. It will always +contain a Dockerfile, and might also contain copies of your local repos, and +other data needed for the build container. +""" + +import os +import tempfile +import textwrap + +from docker_builder import DockerFBCodeBuilder +from parse_args import parse_args_to_fbcode_builder_opts + + +def make_docker_context( + get_steps_fn, github_project, opts=None, default_context_dir=None +): + """ + Returns a path to the Docker context directory. See parse_args.py. + + Helper for making a command-line utility that writes your project's + Dockerfile and associated data into a (temporary) directory. Your main + program might look something like this: + + print(make_docker_context( + lambda builder: [builder.step(...), ...], + 'facebook/your_project', + )) + """ + + if opts is None: + opts = {} + + valid_versions = ( + ("ubuntu:16.04", "5"), + ("ubuntu:18.04", "7"), + ) + + def add_args(parser): + parser.add_argument( + "--docker-context-dir", + metavar="DIR", + default=default_context_dir, + help="Write the Dockerfile and its context into this directory. " + "If empty, make a temporary directory. Default: %(default)s.", + ) + parser.add_argument( + "--user", + metavar="NAME", + default=opts.get("user", "nobody"), + help="Build and install as this user. Default: %(default)s.", + ) + parser.add_argument( + "--prefix", + metavar="DIR", + default=opts.get("prefix", "/home/install"), + help="Install all libraries in this prefix. Default: %(default)s.", + ) + parser.add_argument( + "--projects-dir", + metavar="DIR", + default=opts.get("projects_dir", "/home"), + help="Place project code directories here. Default: %(default)s.", + ) + parser.add_argument( + "--os-image", + metavar="IMG", + choices=zip(*valid_versions)[0], + default=opts.get("os_image", valid_versions[0][0]), + help="Docker OS image -- be sure to use only ones you trust (See " + "README.docker). Choices: %(choices)s. Default: %(default)s.", + ) + parser.add_argument( + "--gcc-version", + metavar="VER", + choices=set(zip(*valid_versions)[1]), + default=opts.get("gcc_version", valid_versions[0][1]), + help="Choices: %(choices)s. Default: %(default)s.", + ) + parser.add_argument( + "--make-parallelism", + metavar="NUM", + type=int, + default=opts.get("make_parallelism", 1), + help="Use `make -j` on multi-CPU systems with lots of RAM. " + "Default: %(default)s.", + ) + parser.add_argument( + "--local-repo-dir", + metavar="DIR", + help="If set, build {0} from a local directory instead of Github.".format( + github_project + ), + ) + parser.add_argument( + "--ccache-tgz", + metavar="PATH", + help="If set, enable ccache for the build. To initialize the " + "cache, first try to hardlink, then to copy --cache-tgz " + "as ccache.tgz into the --docker-context-dir.", + ) + + opts = parse_args_to_fbcode_builder_opts( + add_args, + # These have add_argument() calls, others are set via --option. + ( + "docker_context_dir", + "user", + "prefix", + "projects_dir", + "os_image", + "gcc_version", + "make_parallelism", + "local_repo_dir", + "ccache_tgz", + ), + opts, + help=textwrap.dedent( + """ + + Reads `fbcode_builder_config.py` from the current directory, and + prepares a Docker context directory to build {github_project} and + its dependencies. Prints to stdout the path to the context + directory. + + Pass --option {github_project}:git_hash SHA1 to build something + other than the master branch from Github. + + Or, pass --option {github_project}:local_repo_dir LOCAL_PATH to + build from a local repo instead of cloning from Github. + + Usage: + (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log) + + """.format( + github_project=github_project + ) + ), + ) + + # This allows travis_docker_build.sh not to know the main Github project. + local_repo_dir = opts.pop("local_repo_dir", None) + if local_repo_dir is not None: + opts["{0}:local_repo_dir".format(github_project)] = local_repo_dir + + if (opts.get("os_image"), opts.get("gcc_version")) not in valid_versions: + raise Exception( + "Due to 4/5 ABI changes (std::string), we can only use {0}".format( + " / ".join("GCC {1} on {0}".format(*p) for p in valid_versions) + ) + ) + + if opts.get("docker_context_dir") is None: + opts["docker_context_dir"] = tempfile.mkdtemp(prefix="docker-context-") + elif not os.path.exists(opts.get("docker_context_dir")): + os.makedirs(opts.get("docker_context_dir")) + + builder = DockerFBCodeBuilder(**opts) + context_dir = builder.option("docker_context_dir") # Mark option "in-use" + # The renderer may also populate some files into the context_dir. + dockerfile = builder.render(get_steps_fn(builder)) + + with os.fdopen( + os.open( + os.path.join(context_dir, "Dockerfile"), + os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files + 0o644, + ), + "w", + ) as f: + f.write(dockerfile) + + return context_dir + + +if __name__ == "__main__": + from utils import read_fbcode_builder_config, build_fbcode_builder_config + + # Load a spec from the current directory + config = read_fbcode_builder_config("fbcode_builder_config.py") + print( + make_docker_context( + build_fbcode_builder_config(config), + config["github_project"], + ) + ) diff --git a/build/fbcode_builder/manifests/CLI11 b/build/fbcode_builder/manifests/CLI11 new file mode 100644 index 000000000000..ad161bde1c81 --- /dev/null +++ b/build/fbcode_builder/manifests/CLI11 @@ -0,0 +1,14 @@ +[manifest] +name = CLI11 + +[download] +url = https://github.com/CLIUtils/CLI11/archive/v1.9.0.tar.gz +sha256 = 67640f37ec3be9289039930c987a492badc600645b65057023679f7bb99734e4 + +[build] +builder = cmake +subdir = CLI11-1.9.0 + +[cmake.defines] +CLI11_BUILD_TESTS = OFF +CLI11_BUILD_EXAMPLES = OFF diff --git a/build/fbcode_builder/manifests/OpenNSA b/build/fbcode_builder/manifests/OpenNSA new file mode 100644 index 000000000000..62354c997e6b --- /dev/null +++ b/build/fbcode_builder/manifests/OpenNSA @@ -0,0 +1,17 @@ +[manifest] +name = OpenNSA + +[download] +url = https://docs.broadcom.com/docs-and-downloads/csg/opennsa-6.5.22.tgz +sha256 = 74bfbdaebb6bfe9ebb0deac3aff624385cdcf5aa416ba63706c36538b3c3c46c + +[build] +builder = nop +subdir = opennsa-6.5.22 + +[install.files] +lib/x86-64 = lib +include = include +src/gpl-modules/systems/bde/linux/include = include/systems/bde/linux +src/gpl-modules/include/ibde.h = include/ibde.h +src/gpl-modules = src/gpl-modules diff --git a/build/fbcode_builder/manifests/autoconf b/build/fbcode_builder/manifests/autoconf new file mode 100644 index 000000000000..35963096c534 --- /dev/null +++ b/build/fbcode_builder/manifests/autoconf @@ -0,0 +1,16 @@ +[manifest] +name = autoconf + +[rpms] +autoconf + +[debs] +autoconf + +[download] +url = http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz +sha256 = 954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 + +[build] +builder = autoconf +subdir = autoconf-2.69 diff --git a/build/fbcode_builder/manifests/automake b/build/fbcode_builder/manifests/automake new file mode 100644 index 000000000000..71115068a46c --- /dev/null +++ b/build/fbcode_builder/manifests/automake @@ -0,0 +1,19 @@ +[manifest] +name = automake + +[rpms] +automake + +[debs] +automake + +[download] +url = http://ftp.gnu.org/gnu/automake/automake-1.16.1.tar.gz +sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8 + +[build] +builder = autoconf +subdir = automake-1.16.1 + +[dependencies] +autoconf diff --git a/build/fbcode_builder/manifests/bison b/build/fbcode_builder/manifests/bison new file mode 100644 index 000000000000..6e355d05274a --- /dev/null +++ b/build/fbcode_builder/manifests/bison @@ -0,0 +1,27 @@ +[manifest] +name = bison + +[rpms] +bison + +[debs] +bison + +[download.not(os=windows)] +url = https://mirrors.kernel.org/gnu/bison/bison-3.3.tar.gz +sha256 = fdeafb7fffade05604a61e66b8c040af4b2b5cbb1021dcfe498ed657ac970efd + +[download.os=windows] +url = https://github.com/lexxmark/winflexbison/releases/download/v2.5.17/winflexbison-2.5.17.zip +sha256 = 3dc27a16c21b717bcc5de8590b564d4392a0b8577170c058729d067d95ded825 + +[build.not(os=windows)] +builder = autoconf +subdir = bison-3.3 + +[build.os=windows] +builder = nop + +[install.files.os=windows] +data = bin/data +win_bison.exe = bin/bison.exe diff --git a/build/fbcode_builder/manifests/bistro b/build/fbcode_builder/manifests/bistro new file mode 100644 index 000000000000..d93839275aeb --- /dev/null +++ b/build/fbcode_builder/manifests/bistro @@ -0,0 +1,28 @@ +[manifest] +name = bistro +fbsource_path = fbcode/bistro +shipit_project = bistro +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/bistro.git + +[build.os=linux] +builder = bistro + +# Bistro is Linux-specific +[build.not(os=linux)] +builder = nop + +[dependencies] +fmt +folly +proxygen +fbthrift +libsodium +googletest_1_8 +sqlite3 + +[shipit.pathmap] +fbcode/bistro/public_tld = . +fbcode/bistro = bistro diff --git a/build/fbcode_builder/manifests/boost b/build/fbcode_builder/manifests/boost new file mode 100644 index 000000000000..4b254e308ab2 --- /dev/null +++ b/build/fbcode_builder/manifests/boost @@ -0,0 +1,86 @@ +[manifest] +name = boost + +[download.not(os=windows)] +url = https://versaweb.dl.sourceforge.net/project/boost/boost/1.69.0/boost_1_69_0.tar.bz2 +sha256 = 8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406 + +[download.os=windows] +url = https://versaweb.dl.sourceforge.net/project/boost/boost/1.69.0/boost_1_69_0.zip +sha256 = d074bcbcc0501c4917b965fc890e303ee70d8b01ff5712bae4a6c54f2b6b4e52 + +[preinstalled.env] +BOOST_ROOT_1_69_0 + +[debs] +libboost-all-dev + +[rpms] +boost +boost-math +boost-test +boost-fiber +boost-graph +boost-log +boost-openmpi +boost-timer +boost-chrono +boost-locale +boost-thread +boost-atomic +boost-random +boost-static +boost-contract +boost-date-time +boost-iostreams +boost-container +boost-coroutine +boost-filesystem +boost-system +boost-stacktrace +boost-regex +boost-devel +boost-context +boost-python3-devel +boost-type_erasure +boost-wave +boost-python3 +boost-serialization +boost-program-options + +[build] +builder = boost + +[b2.args] +--with-atomic +--with-chrono +--with-container +--with-context +--with-contract +--with-coroutine +--with-date_time +--with-exception +--with-fiber +--with-filesystem +--with-graph +--with-graph_parallel +--with-iostreams +--with-locale +--with-log +--with-math +--with-mpi +--with-program_options +--with-python +--with-random +--with-regex +--with-serialization +--with-stacktrace +--with-system +--with-test +--with-thread +--with-timer +--with-type_erasure +--with-wave + +[b2.args.os=darwin] +toolset=clang diff --git a/build/fbcode_builder/manifests/cmake b/build/fbcode_builder/manifests/cmake new file mode 100644 index 000000000000..f756caed09f4 --- /dev/null +++ b/build/fbcode_builder/manifests/cmake @@ -0,0 +1,43 @@ +[manifest] +name = cmake + +[rpms] +cmake + +# All current deb based distros have a cmake that is too old +#[debs] +#cmake + +[dependencies] +ninja + +[download.os=windows] +url = https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-win64-x64.zip +sha256 = 40e8140d68120378262322bbc8c261db8d184d7838423b2e5bf688a6209d3807 + +[download.os=darwin] +url = https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Darwin-x86_64.tar.gz +sha256 = a02ad0d5b955dfad54c095bd7e937eafbbbfe8a99860107025cc442290a3e903 + +[download.os=linux] +url = https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0.tar.gz +sha256 = aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 + +[build.os=windows] +builder = nop +subdir = cmake-3.14.0-win64-x64 + +[build.os=darwin] +builder = nop +subdir = cmake-3.14.0-Darwin-x86_64 + +[install.files.os=darwin] +CMake.app/Contents/bin = bin +CMake.app/Contents/share = share + +[build.os=linux] +builder = cmakebootstrap +subdir = cmake-3.14.0 + +[make.install_args.os=linux] +install diff --git a/build/fbcode_builder/manifests/cpptoml b/build/fbcode_builder/manifests/cpptoml new file mode 100644 index 000000000000..5a3c781dc708 --- /dev/null +++ b/build/fbcode_builder/manifests/cpptoml @@ -0,0 +1,10 @@ +[manifest] +name = cpptoml + +[download] +url = https://github.com/skystrife/cpptoml/archive/v0.1.1.tar.gz +sha256 = 23af72468cfd4040984d46a0dd2a609538579c78ddc429d6b8fd7a10a6e24403 + +[build] +builder = cmake +subdir = cpptoml-0.1.1 diff --git a/build/fbcode_builder/manifests/double-conversion b/build/fbcode_builder/manifests/double-conversion new file mode 100644 index 000000000000..e27c7ae068df --- /dev/null +++ b/build/fbcode_builder/manifests/double-conversion @@ -0,0 +1,11 @@ +[manifest] +name = double-conversion + +[download] +url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz +sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021 + +[build] +builder = cmake +subdir = double-conversion-3.1.4 + diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden new file mode 100644 index 000000000000..3174bb3df1b5 --- /dev/null +++ b/build/fbcode_builder/manifests/eden @@ -0,0 +1,69 @@ +[manifest] +name = eden +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/eden.git + +[build] +builder = cmake + +[dependencies] +googletest +folly +fbthrift +fb303 +cpptoml +rocksdb +re2 +libgit2 +lz4 +pexpect +python-toml + +[dependencies.fb=on] +rust + +# macOS ships with sqlite3, and some of the core system +# frameworks require that that version be linked rather +# than the one we might build for ourselves here, so we +# skip building it on macos. +[dependencies.not(os=darwin)] +sqlite3 + +[dependencies.os=darwin] +osxfuse + +# TODO: teach getdeps to compile curl on Windows. +# Enabling curl on Windows requires us to find a way to compile libcurl with +# msvc. +[dependencies.not(os=windows)] +libcurl + +[shipit.pathmap] +fbcode/eden/oss = . +fbcode/eden = eden +fbcode/tools/lfs = tools/lfs +fbcode/thrift/lib/rust = thrift/lib/rust + +[shipit.strip] +^fbcode/eden/fs/eden-config\.h$ +^fbcode/eden/fs/py/eden/config\.py$ +^fbcode/eden/hg/.*$ +^fbcode/eden/mononoke/(?!lfs_protocol) +^fbcode/eden/scm/build/.*$ +^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ +^fbcode/eden/.*/\.cargo/.*$ +/Cargo\.lock$ +\.pyc$ + +[cmake.defines.all(fb=on,os=windows)] +INSTALL_PYTHON_LIB=ON + +[cmake.defines.fb=on] +USE_CARGO_VENDOR=ON + +[depends.environment] +EDEN_VERSION_OVERRIDE diff --git a/build/fbcode_builder/manifests/eden_scm b/build/fbcode_builder/manifests/eden_scm new file mode 100644 index 000000000000..cfe9c709660b --- /dev/null +++ b/build/fbcode_builder/manifests/eden_scm @@ -0,0 +1,57 @@ +[manifest] +name = eden_scm +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/eden.git + +[build.not(os=windows)] +builder = make +subdir = eden/scm +disable_env_override_pkgconfig = 1 +disable_env_override_path = 1 + +[build.os=windows] +# For now the biggest blocker is missing "make" on windows, but there are bound +# to be more +builder = nop + +[make.build_args] +getdepsbuild + +[make.install_args] +install-getdeps + +[make.test_args] +test-getdeps + +[shipit.pathmap] +fbcode/common/rust = common/rust +fbcode/eden/oss = . +fbcode/eden = eden +fbcode/tools/lfs = tools/lfs +fbcode/fboss/common = common + +[shipit.strip] +^fbcode/eden/fs/eden-config\.h$ +^fbcode/eden/fs/py/eden/config\.py$ +^fbcode/eden/hg/.*$ +^fbcode/eden/mononoke/(?!lfs_protocol) +^fbcode/eden/scm/build/.*$ +^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ +^fbcode/eden/.*/\.cargo/.*$ +^.*/fb/.*$ +/Cargo\.lock$ +\.pyc$ + +[dependencies] +fb303-source +fbthrift +fbthrift-source +openssl +rust-shed + +[dependencies.fb=on] +rust diff --git a/build/fbcode_builder/manifests/eden_scm_lib_edenapi_tools b/build/fbcode_builder/manifests/eden_scm_lib_edenapi_tools new file mode 100644 index 000000000000..be29d70f8d3e --- /dev/null +++ b/build/fbcode_builder/manifests/eden_scm_lib_edenapi_tools @@ -0,0 +1,36 @@ +[manifest] +name = eden_scm_lib_edenapi_tools +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/eden.git + +[build] +builder = cargo + +[cargo] +build_doc = true +manifests_to_build = eden/scm/lib/edenapi/tools/make_req/Cargo.toml,eden/scm/lib/edenapi/tools/read_res/Cargo.toml + +[shipit.pathmap] +fbcode/eden/oss = . +fbcode/eden = eden +fbcode/tools/lfs = tools/lfs +fbcode/fboss/common = common + +[shipit.strip] +^fbcode/eden/fs/eden-config\.h$ +^fbcode/eden/fs/py/eden/config\.py$ +^fbcode/eden/hg/.*$ +^fbcode/eden/mononoke/(?!lfs_protocol) +^fbcode/eden/scm/build/.*$ +^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ +^fbcode/eden/.*/\.cargo/.*$ +^.*/fb/.*$ +/Cargo\.lock$ +\.pyc$ + +[dependencies.fb=on] +rust diff --git a/build/fbcode_builder/manifests/f4d b/build/fbcode_builder/manifests/f4d new file mode 100644 index 000000000000..db30894c7ba8 --- /dev/null +++ b/build/fbcode_builder/manifests/f4d @@ -0,0 +1,29 @@ +[manifest] +name = f4d +fbsource_path = fbcode/f4d +shipit_project = f4d +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexternal/f4d.git + +[build.os=windows] +builder = nop + +[build.not(os=windows)] +builder = cmake + +[dependencies] +double-conversion +folly +glog +googletest +boost +protobuf +lzo +libicu +re2 + +[shipit.pathmap] +fbcode/f4d/public_tld = . +fbcode/f4d = f4d diff --git a/build/fbcode_builder/manifests/fatal b/build/fbcode_builder/manifests/fatal new file mode 100644 index 000000000000..3c333561f8ac --- /dev/null +++ b/build/fbcode_builder/manifests/fatal @@ -0,0 +1,15 @@ +[manifest] +name = fatal +fbsource_path = fbcode/fatal +shipit_project = fatal + +[git] +repo_url = https://github.com/facebook/fatal.git + +[shipit.pathmap] +fbcode/fatal = . +fbcode/fatal/public_tld = . + +[build] +builder = nop +subdir = . diff --git a/build/fbcode_builder/manifests/fb303 b/build/fbcode_builder/manifests/fb303 new file mode 100644 index 000000000000..743aca01ecfe --- /dev/null +++ b/build/fbcode_builder/manifests/fb303 @@ -0,0 +1,27 @@ +[manifest] +name = fb303 +fbsource_path = fbcode/fb303 +shipit_project = fb303 +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookincubator/fb303.git + +[build] +builder = cmake + +[dependencies] +folly +gflags +glog +fbthrift + +[cmake.defines.test=on] +BUILD_TESTS=ON + +[cmake.defines.test=off] +BUILD_TESTS=OFF + +[shipit.pathmap] +fbcode/fb303/github = . +fbcode/fb303 = fb303 diff --git a/build/fbcode_builder/manifests/fb303-source b/build/fbcode_builder/manifests/fb303-source new file mode 100644 index 000000000000..ea160c50015a --- /dev/null +++ b/build/fbcode_builder/manifests/fb303-source @@ -0,0 +1,15 @@ +[manifest] +name = fb303-source +fbsource_path = fbcode/fb303 +shipit_project = fb303 +shipit_fbcode_builder = false + +[git] +repo_url = https://github.com/facebook/fb303.git + +[build] +builder = nop + +[shipit.pathmap] +fbcode/fb303/github = . +fbcode/fb303 = fb303 diff --git a/build/fbcode_builder/manifests/fboss b/build/fbcode_builder/manifests/fboss new file mode 100644 index 000000000000..f29873e72843 --- /dev/null +++ b/build/fbcode_builder/manifests/fboss @@ -0,0 +1,42 @@ +[manifest] +name = fboss +fbsource_path = fbcode/fboss +shipit_project = fboss +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/fboss.git + +[build.os=linux] +builder = cmake + +[build.not(os=linux)] +builder = nop + +[dependencies] +folly +fb303 +wangle +fizz +fmt +libsodium +googletest +zstd +fbthrift +iproute2 +libmnl +libusb +libcurl +libnl +libsai +OpenNSA +re2 +python +yaml-cpp +libyaml +CLI11 + +[shipit.pathmap] +fbcode/fboss/github = . +fbcode/fboss/common = common +fbcode/fboss = fboss diff --git a/build/fbcode_builder/manifests/fbthrift b/build/fbcode_builder/manifests/fbthrift new file mode 100644 index 000000000000..072dd451220b --- /dev/null +++ b/build/fbcode_builder/manifests/fbthrift @@ -0,0 +1,33 @@ +[manifest] +name = fbthrift +fbsource_path = fbcode/thrift +shipit_project = fbthrift +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/fbthrift.git + +[build] +builder = cmake + +[dependencies] +bison +flex +folly +wangle +fizz +fmt +googletest +libsodium +python-six +zstd + +[shipit.pathmap] +fbcode/thrift/public_tld = . +fbcode/thrift = thrift + +[shipit.strip] +^fbcode/thrift/thrift-config\.h$ +^fbcode/thrift/perf/canary.py$ +^fbcode/thrift/perf/loadtest.py$ +^fbcode/thrift/.castle/.* diff --git a/build/fbcode_builder/manifests/fbthrift-source b/build/fbcode_builder/manifests/fbthrift-source new file mode 100644 index 000000000000..7af0d6ddac0e --- /dev/null +++ b/build/fbcode_builder/manifests/fbthrift-source @@ -0,0 +1,21 @@ +[manifest] +name = fbthrift-source +fbsource_path = fbcode/thrift +shipit_project = fbthrift +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/fbthrift.git + +[build] +builder = nop + +[shipit.pathmap] +fbcode/thrift/public_tld = . +fbcode/thrift = thrift + +[shipit.strip] +^fbcode/thrift/thrift-config\.h$ +^fbcode/thrift/perf/canary.py$ +^fbcode/thrift/perf/loadtest.py$ +^fbcode/thrift/.castle/.* diff --git a/build/fbcode_builder/manifests/fbzmq b/build/fbcode_builder/manifests/fbzmq new file mode 100644 index 000000000000..5739016c84ac --- /dev/null +++ b/build/fbcode_builder/manifests/fbzmq @@ -0,0 +1,29 @@ +[manifest] +name = fbzmq +fbsource_path = facebook/fbzmq +shipit_project = fbzmq +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/fbzmq.git + +[build.os=linux] +builder = cmake + +[build.not(os=linux)] +# boost.fiber is required and that is not available on macos. +# libzmq doesn't currently build on windows. +builder = nop + +[dependencies] +boost +folly +fbthrift +googletest +libzmq + +[shipit.pathmap] +fbcode/fbzmq = fbzmq +fbcode/fbzmq/public_tld = . + +[shipit.strip] diff --git a/build/fbcode_builder/manifests/fizz b/build/fbcode_builder/manifests/fizz new file mode 100644 index 000000000000..72f29973f10d --- /dev/null +++ b/build/fbcode_builder/manifests/fizz @@ -0,0 +1,36 @@ +[manifest] +name = fizz +fbsource_path = fbcode/fizz +shipit_project = fizz +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookincubator/fizz.git + +[build] +builder = cmake +subdir = fizz + +[cmake.defines] +BUILD_EXAMPLES = OFF + +[cmake.defines.test=on] +BUILD_TESTS = ON + +[cmake.defines.all(os=windows, test=on)] +BUILD_TESTS = OFF + +[cmake.defines.test=off] +BUILD_TESTS = OFF + +[dependencies] +folly +libsodium +zstd + +[dependencies.all(test=on, not(os=windows))] +googletest_1_8 + +[shipit.pathmap] +fbcode/fizz/public_tld = . +fbcode/fizz = fizz diff --git a/build/fbcode_builder/manifests/flex b/build/fbcode_builder/manifests/flex new file mode 100644 index 000000000000..f266c4033607 --- /dev/null +++ b/build/fbcode_builder/manifests/flex @@ -0,0 +1,32 @@ +[manifest] +name = flex + +[rpms] +flex + +[debs] +flex + +[download.not(os=windows)] +url = https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz +sha256 = e87aae032bf07c26f85ac0ed3250998c37621d95f8bd748b31f15b33c45ee995 + +[download.os=windows] +url = https://github.com/lexxmark/winflexbison/releases/download/v2.5.17/winflexbison-2.5.17.zip +sha256 = 3dc27a16c21b717bcc5de8590b564d4392a0b8577170c058729d067d95ded825 + +[build.not(os=windows)] +builder = autoconf +subdir = flex-2.6.4 + +[build.os=windows] +builder = nop + +[install.files.os=windows] +data = bin/data +win_flex.exe = bin/flex.exe + +# Moral equivalent to this PR that fixes a crash when bootstrapping flex +# on linux: https://github.com/easybuilders/easybuild-easyconfigs/pull/5792 +[autoconf.args.os=linux] +CFLAGS=-D_GNU_SOURCE diff --git a/build/fbcode_builder/manifests/fmt b/build/fbcode_builder/manifests/fmt new file mode 100644 index 000000000000..21503d202cab --- /dev/null +++ b/build/fbcode_builder/manifests/fmt @@ -0,0 +1,14 @@ +[manifest] +name = fmt + +[download] +url = https://github.com/fmtlib/fmt/archive/6.1.1.tar.gz +sha256 = bf4e50955943c1773cc57821d6c00f7e2b9e10eb435fafdd66739d36056d504e + +[build] +builder = cmake +subdir = fmt-6.1.1 + +[cmake.defines] +FMT_TEST = OFF +FMT_DOC = OFF diff --git a/build/fbcode_builder/manifests/folly b/build/fbcode_builder/manifests/folly new file mode 100644 index 000000000000..9647b17f87dc --- /dev/null +++ b/build/fbcode_builder/manifests/folly @@ -0,0 +1,58 @@ +[manifest] +name = folly +fbsource_path = fbcode/folly +shipit_project = folly +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/folly.git + +[build] +builder = cmake + +[dependencies] +gflags +glog +googletest +boost +libevent +double-conversion +fmt +lz4 +snappy +zstd +# no openssl or zlib in the linux case, why? +# these are usually installed on the system +# and are the easiest system deps to pull in. +# In the future we want to be able to express +# that a system dep is sufficient in the manifest +# for eg: openssl and zlib, but for now we don't +# have it. + +# macOS doesn't expose the openssl api so we need +# to build our own. +[dependencies.os=darwin] +openssl + +# Windows has neither openssl nor zlib, so we get +# to provide both +[dependencies.os=windows] +openssl +zlib + +[shipit.pathmap] +fbcode/folly/public_tld = . +fbcode/folly = folly + +[shipit.strip] +^fbcode/folly/folly-config\.h$ +^fbcode/folly/public_tld/build/facebook_.* + +[cmake.defines] +BUILD_SHARED_LIBS=OFF + +[cmake.defines.test=on] +BUILD_TESTS=ON + +[cmake.defines.test=off] +BUILD_TESTS=OFF diff --git a/build/fbcode_builder/manifests/gflags b/build/fbcode_builder/manifests/gflags new file mode 100644 index 000000000000..d7ec44eab735 --- /dev/null +++ b/build/fbcode_builder/manifests/gflags @@ -0,0 +1,17 @@ +[manifest] +name = gflags + +[download] +url = https://github.com/gflags/gflags/archive/v2.2.2.tar.gz +sha256 = 34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf + +[build] +builder = cmake +subdir = gflags-2.2.2 + +[cmake.defines] +BUILD_SHARED_LIBS = ON +BUILD_STATIC_LIBS = ON +#BUILD_gflags_nothreads_LIB = OFF +BUILD_gflags_LIB = ON + diff --git a/build/fbcode_builder/manifests/git-lfs b/build/fbcode_builder/manifests/git-lfs new file mode 100644 index 000000000000..38a5e6aeba58 --- /dev/null +++ b/build/fbcode_builder/manifests/git-lfs @@ -0,0 +1,12 @@ +[manifest] +name = git-lfs + +[download.os=linux] +url = https://github.com/git-lfs/git-lfs/releases/download/v2.9.1/git-lfs-linux-amd64-v2.9.1.tar.gz +sha256 = 2a8e60cf51ec45aa0f4332aa0521d60ec75c76e485d13ebaeea915b9d70ea466 + +[build] +builder = nop + +[install.files] +git-lfs = bin/git-lfs diff --git a/build/fbcode_builder/manifests/glog b/build/fbcode_builder/manifests/glog new file mode 100644 index 000000000000..d2354610ac45 --- /dev/null +++ b/build/fbcode_builder/manifests/glog @@ -0,0 +1,16 @@ +[manifest] +name = glog + +[download] +url = https://github.com/google/glog/archive/v0.4.0.tar.gz +sha256 = f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c + +[build] +builder = cmake +subdir = glog-0.4.0 + +[dependencies] +gflags + +[cmake.defines] +BUILD_SHARED_LIBS=ON diff --git a/build/fbcode_builder/manifests/gnu-bash b/build/fbcode_builder/manifests/gnu-bash new file mode 100644 index 000000000000..89da77ca2b70 --- /dev/null +++ b/build/fbcode_builder/manifests/gnu-bash @@ -0,0 +1,20 @@ +[manifest] +name = gnu-bash + +[download.os=darwin] +url = https://ftp.gnu.org/gnu/bash/bash-5.1-rc1.tar.gz +sha256 = 0b2684eb1990329d499c96decfe2459f3e150deb915b0a9d03cf1be692b1d6d3 + +[build.os=darwin] +# The buildin FreeBSD bash on OSX is both outdated and incompatible with the +# modern GNU bash, so for the sake of being cross-platform friendly this +# manifest provides GNU bash. +# NOTE: This is the 5.1-rc1 version, which is almost the same as what Homebrew +# uses (Homebrew installs 5.0 with the 18 patches that in fact make the 5.1-rc1 +# version). +builder = autoconf +subdir = bash-5.1-rc1 +build_in_src_dir = true + +[build.not(os=darwin)] +builder = nop diff --git a/build/fbcode_builder/manifests/gnu-coreutils b/build/fbcode_builder/manifests/gnu-coreutils new file mode 100644 index 000000000000..1ab4d9d4a5a5 --- /dev/null +++ b/build/fbcode_builder/manifests/gnu-coreutils @@ -0,0 +1,15 @@ +[manifest] +name = gnu-coreutils + +[download.os=darwin] +url = https://ftp.gnu.org/gnu/coreutils/coreutils-8.32.tar.gz +sha256 = d5ab07435a74058ab69a2007e838be4f6a90b5635d812c2e26671e3972fca1b8 + +[build.os=darwin] +# The buildin FreeBSD version incompatible with the GNU one, so for the sake of +# being cross-platform friendly this manifest provides the GNU version. +builder = autoconf +subdir = coreutils-8.32 + +[build.not(os=darwin)] +builder = nop diff --git a/build/fbcode_builder/manifests/gnu-grep b/build/fbcode_builder/manifests/gnu-grep new file mode 100644 index 000000000000..e6a163d37a84 --- /dev/null +++ b/build/fbcode_builder/manifests/gnu-grep @@ -0,0 +1,15 @@ +[manifest] +name = gnu-grep + +[download.os=darwin] +url = https://ftp.gnu.org/gnu/grep/grep-3.5.tar.gz +sha256 = 9897220992a8fd38a80b70731462defa95f7ff2709b235fb54864ddd011141dd + +[build.os=darwin] +# The buildin FreeBSD version incompatible with the GNU one, so for the sake of +# being cross-platform friendly this manifest provides the GNU version. +builder = autoconf +subdir = grep-3.5 + +[build.not(os=darwin)] +builder = nop diff --git a/build/fbcode_builder/manifests/gnu-sed b/build/fbcode_builder/manifests/gnu-sed new file mode 100644 index 000000000000..9b458df6ef98 --- /dev/null +++ b/build/fbcode_builder/manifests/gnu-sed @@ -0,0 +1,15 @@ +[manifest] +name = gnu-sed + +[download.os=darwin] +url = https://ftp.gnu.org/gnu/sed/sed-4.8.tar.gz +sha256 = 53cf3e14c71f3a149f29d13a0da64120b3c1d3334fba39c4af3e520be053982a + +[build.os=darwin] +# The buildin FreeBSD version incompatible with the GNU one, so for the sake of +# being cross-platform friendly this manifest provides the GNU version. +builder = autoconf +subdir = sed-4.8 + +[build.not(os=darwin)] +builder = nop diff --git a/build/fbcode_builder/manifests/googletest b/build/fbcode_builder/manifests/googletest new file mode 100644 index 000000000000..775aac34f0de --- /dev/null +++ b/build/fbcode_builder/manifests/googletest @@ -0,0 +1,18 @@ +[manifest] +name = googletest + +[download] +url = https://github.com/google/googletest/archive/release-1.10.0.tar.gz +sha256 = 9dc9157a9a1551ec7a7e43daea9a694a0bb5fb8bec81235d8a1e6ef64c716dcb + +[build] +builder = cmake +subdir = googletest-release-1.10.0 + +[cmake.defines] +# Everything else defaults to the shared runtime, so tell gtest that +# it should not use its choice of the static runtime +gtest_force_shared_crt=ON + +[cmake.defines.os=windows] +BUILD_SHARED_LIBS=ON diff --git a/build/fbcode_builder/manifests/googletest_1_8 b/build/fbcode_builder/manifests/googletest_1_8 new file mode 100644 index 000000000000..76c0ce51f9eb --- /dev/null +++ b/build/fbcode_builder/manifests/googletest_1_8 @@ -0,0 +1,18 @@ +[manifest] +name = googletest_1_8 + +[download] +url = https://github.com/google/googletest/archive/release-1.8.0.tar.gz +sha256 = 58a6f4277ca2bc8565222b3bbd58a177609e9c488e8a72649359ba51450db7d8 + +[build] +builder = cmake +subdir = googletest-release-1.8.0 + +[cmake.defines] +# Everything else defaults to the shared runtime, so tell gtest that +# it should not use its choice of the static runtime +gtest_force_shared_crt=ON + +[cmake.defines.os=windows] +BUILD_SHARED_LIBS=ON diff --git a/build/fbcode_builder/manifests/gperf b/build/fbcode_builder/manifests/gperf new file mode 100644 index 000000000000..13d7a890fded --- /dev/null +++ b/build/fbcode_builder/manifests/gperf @@ -0,0 +1,14 @@ +[manifest] +name = gperf + +[download] +url = http://ftp.gnu.org/pub/gnu/gperf/gperf-3.1.tar.gz +sha256 = 588546b945bba4b70b6a3a616e80b4ab466e3f33024a352fc2198112cdbb3ae2 + +[build.not(os=windows)] +builder = autoconf +subdir = gperf-3.1 + +[build.os=windows] +builder = nop + diff --git a/build/fbcode_builder/manifests/iproute2 b/build/fbcode_builder/manifests/iproute2 new file mode 100644 index 000000000000..6fb7f77ed9c2 --- /dev/null +++ b/build/fbcode_builder/manifests/iproute2 @@ -0,0 +1,13 @@ +[manifest] +name = iproute2 + +[download] +url = https://mirrors.edge.kernel.org/pub/linux/utils/net/iproute2/iproute2-4.12.0.tar.gz +sha256 = 46612a1e2d01bb31932557bccdb1b8618cae9a439dfffc08ef35ed8e197f14ce + +[build.os=linux] +builder = iproute2 +subdir = iproute2-4.12.0 + +[build.not(os=linux)] +builder = nop diff --git a/build/fbcode_builder/manifests/jq b/build/fbcode_builder/manifests/jq new file mode 100644 index 000000000000..231818f343e9 --- /dev/null +++ b/build/fbcode_builder/manifests/jq @@ -0,0 +1,24 @@ +[manifest] +name = jq + +[rpms] +jq + +[debs] +jq + +[download.not(os=windows)] +url = https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz +sha256 = c4d2bfec6436341113419debf479d833692cc5cdab7eb0326b5a4d4fbe9f493c + +[build.not(os=windows)] +builder = autoconf +subdir = jq-1.5 + +[build.os=windows] +builder = nop + +[autoconf.args] +# This argument turns off some developers tool and it is recommended in jq's +# README +--disable-maintainer-mode diff --git a/build/fbcode_builder/manifests/katran b/build/fbcode_builder/manifests/katran new file mode 100644 index 000000000000..224ccbe2157f --- /dev/null +++ b/build/fbcode_builder/manifests/katran @@ -0,0 +1,38 @@ +[manifest] +name = katran +fbsource_path = fbcode/katran +shipit_project = katran +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookincubator/katran.git + +[build.not(os=linux)] +builder = nop + +[build.os=linux] +builder = cmake +subdir = . + +[cmake.defines.test=on] +BUILD_TESTS=ON + +[cmake.defines.test=off] +BUILD_TESTS=OFF + +[dependencies] +folly +fizz +libbpf +libmnl +zlib +googletest + + +[shipit.pathmap] +fbcode/katran/public_root = . +fbcode/katran = katran + +[shipit.strip] +^fbcode/katran/facebook +^fbcode/katran/OSS_SYNC diff --git a/build/fbcode_builder/manifests/libbpf b/build/fbcode_builder/manifests/libbpf new file mode 100644 index 000000000000..0416822e4346 --- /dev/null +++ b/build/fbcode_builder/manifests/libbpf @@ -0,0 +1,26 @@ +[manifest] +name = libbpf + +[download] +url = https://github.com/libbpf/libbpf/archive/v0.3.tar.gz +sha256 = c168d84a75b541f753ceb49015d9eb886e3fb5cca87cdd9aabce7e10ad3a1efc + +# BPF only builds on linux, so make it a NOP on other platforms +[build.not(os=linux)] +builder = nop + +[build.os=linux] +builder = make +subdir = libbpf-0.3/src + +[make.build_args] +BUILD_STATIC_ONLY=y + +# libbpf-0.3 requires uapi headers >= 5.8 +[make.install_args] +install +install_uapi_headers +BUILD_STATIC_ONLY=y + +[dependencies] +libelf diff --git a/build/fbcode_builder/manifests/libbpf_0_2_0_beta b/build/fbcode_builder/manifests/libbpf_0_2_0_beta new file mode 100644 index 000000000000..072639817d76 --- /dev/null +++ b/build/fbcode_builder/manifests/libbpf_0_2_0_beta @@ -0,0 +1,26 @@ +[manifest] +name = libbpf_0_2_0_beta + +[download] +url = https://github.com/libbpf/libbpf/archive/b6dd2f2.tar.gz +sha256 = 8db9dca90f5c445ef2362e3c6a00f3d6c4bf36e8782f8e27704109c78e541497 + +# BPF only builds on linux, so make it a NOP on other platforms +[build.not(os=linux)] +builder = nop + +[build.os=linux] +builder = make +subdir = libbpf-b6dd2f2b7df4d3bd35d64aaf521d9ad18d766f53/src + +[make.build_args] +BUILD_STATIC_ONLY=y + +# libbpf now requires uapi headers >= 5.8 +[make.install_args] +install +install_uapi_headers +BUILD_STATIC_ONLY=y + +[dependencies] +libelf diff --git a/build/fbcode_builder/manifests/libcurl b/build/fbcode_builder/manifests/libcurl new file mode 100644 index 000000000000..466b4497c35d --- /dev/null +++ b/build/fbcode_builder/manifests/libcurl @@ -0,0 +1,39 @@ +[manifest] +name = libcurl + +[rpms] +libcurl-devel +libcurl + +[debs] +libcurl4-openssl-dev + +[download] +url = https://curl.haxx.se/download/curl-7.65.1.tar.gz +sha256 = 821aeb78421375f70e55381c9ad2474bf279fc454b791b7e95fc83562951c690 + +[dependencies] +nghttp2 + +# We use system OpenSSL on Linux (see folly's manifest for details) +[dependencies.not(os=linux)] +openssl + +[build.not(os=windows)] +builder = autoconf +subdir = curl-7.65.1 + +[autoconf.args] +# fboss (which added the libcurl dep) doesn't need ldap so it is disabled here. +# if someone in the future wants to add ldap for something else, it won't hurt +# fboss. However, that would require adding an ldap manifest. +# +# For the same reason, we disable libssh2 and libidn2 which aren't really used +# but would require adding manifests if we don't disable them. +--disable-ldap +--without-libssh2 +--without-libidn2 + +[build.os=windows] +builder = cmake +subdir = curl-7.65.1 diff --git a/build/fbcode_builder/manifests/libelf b/build/fbcode_builder/manifests/libelf new file mode 100644 index 000000000000..a46aab8796ea --- /dev/null +++ b/build/fbcode_builder/manifests/libelf @@ -0,0 +1,20 @@ +[manifest] +name = libelf + +[rpms] +elfutils-libelf-devel-static + +[debs] +libelf-dev + +[download] +url = https://ftp.osuosl.org/pub/blfs/conglomeration/libelf/libelf-0.8.13.tar.gz +sha256 = 591a9b4ec81c1f2042a97aa60564e0cb79d041c52faa7416acb38bc95bd2c76d + +# libelf only makes sense on linux, so make it a NOP on other platforms +[build.not(os=linux)] +builder = nop + +[build.os=linux] +builder = autoconf +subdir = libelf-0.8.13 diff --git a/build/fbcode_builder/manifests/libevent b/build/fbcode_builder/manifests/libevent new file mode 100644 index 000000000000..eaa39a9e66c9 --- /dev/null +++ b/build/fbcode_builder/manifests/libevent @@ -0,0 +1,29 @@ +[manifest] +name = libevent + +[rpms] +libevent-devel + +[debs] +libevent-dev + +# Note that the CMakeLists.txt file is present only in +# git repo and not in the release tarball, so take care +# to use the github generated source tarball rather than +# the explicitly uploaded source tarball +[download] +url = https://github.com/libevent/libevent/archive/release-2.1.8-stable.tar.gz +sha256 = 316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d + +[build] +builder = cmake +subdir = libevent-release-2.1.8-stable + +[cmake.defines] +EVENT__DISABLE_TESTS = ON +EVENT__DISABLE_BENCHMARK = ON +EVENT__DISABLE_SAMPLES = ON +EVENT__DISABLE_REGRESS = ON + +[dependencies.not(os=linux)] +openssl diff --git a/build/fbcode_builder/manifests/libgit2 b/build/fbcode_builder/manifests/libgit2 new file mode 100644 index 000000000000..1d6a53e5eb51 --- /dev/null +++ b/build/fbcode_builder/manifests/libgit2 @@ -0,0 +1,24 @@ +[manifest] +name = libgit2 + +[rpms] +libgit2-devel + +[debs] +libgit2-dev + +[download] +url = https://github.com/libgit2/libgit2/archive/v0.28.1.tar.gz +sha256 = 0ca11048795b0d6338f2e57717370208c2c97ad66c6d5eac0c97a8827d13936b + +[build] +builder = cmake +subdir = libgit2-0.28.1 + +[cmake.defines] +# Could turn this on if we also wanted to add a manifest for libssh2 +USE_SSH = OFF +BUILD_CLAR = OFF +# Have to build shared to work around annoying problems with cmake +# mis-parsing the frameworks required to link this on macos :-/ +BUILD_SHARED_LIBS = ON diff --git a/build/fbcode_builder/manifests/libicu b/build/fbcode_builder/manifests/libicu new file mode 100644 index 000000000000..c1deda503760 --- /dev/null +++ b/build/fbcode_builder/manifests/libicu @@ -0,0 +1,19 @@ +[manifest] +name = libicu + +[rpms] +libicu-devel + +[debs] +libicu-dev + +[download] +url = https://github.com/unicode-org/icu/releases/download/release-68-2/icu4c-68_2-src.tgz +sha256 = c79193dee3907a2199b8296a93b52c5cb74332c26f3d167269487680d479d625 + +[build.not(os=windows)] +builder = autoconf +subdir = icu/source + +[build.os=windows] +builder = nop diff --git a/build/fbcode_builder/manifests/libmnl b/build/fbcode_builder/manifests/libmnl new file mode 100644 index 000000000000..9b28b87b96d8 --- /dev/null +++ b/build/fbcode_builder/manifests/libmnl @@ -0,0 +1,17 @@ +[manifest] +name = libmnl + +[rpms] +libmnl-devel +libmnl-static + +[debs] +libmnl-dev + +[download] +url = http://www.netfilter.org/pub/libmnl/libmnl-1.0.4.tar.bz2 +sha256 = 171f89699f286a5854b72b91d06e8f8e3683064c5901fb09d954a9ab6f551f81 + +[build.os=linux] +builder = autoconf +subdir = libmnl-1.0.4 diff --git a/build/fbcode_builder/manifests/libnl b/build/fbcode_builder/manifests/libnl new file mode 100644 index 000000000000..f864acb498ca --- /dev/null +++ b/build/fbcode_builder/manifests/libnl @@ -0,0 +1,17 @@ +[manifest] +name = libnl + +[rpms] +libnl3-devel +libnl3 + +[debs] +libnl-3-dev + +[download] +url = https://www.infradead.org/~tgr/libnl/files/libnl-3.2.25.tar.gz +sha256 = 8beb7590674957b931de6b7f81c530b85dc7c1ad8fbda015398bc1e8d1ce8ec5 + +[build.os=linux] +builder = autoconf +subdir = libnl-3.2.25 diff --git a/build/fbcode_builder/manifests/libsai b/build/fbcode_builder/manifests/libsai new file mode 100644 index 000000000000..4f422d8e15f6 --- /dev/null +++ b/build/fbcode_builder/manifests/libsai @@ -0,0 +1,13 @@ +[manifest] +name = libsai + +[download] +url = https://github.com/opencomputeproject/SAI/archive/v1.7.1.tar.gz +sha256 = e18eb1a2a6e5dd286d97e13569d8b78cc1f8229030beed0db4775b9a50ab6a83 + +[build] +builder = nop +subdir = SAI-1.7.1 + +[install.files] +inc = include diff --git a/build/fbcode_builder/manifests/libsodium b/build/fbcode_builder/manifests/libsodium new file mode 100644 index 000000000000..d69bfcc4b3f3 --- /dev/null +++ b/build/fbcode_builder/manifests/libsodium @@ -0,0 +1,33 @@ +[manifest] +name = libsodium + +[rpms] +libsodium-devel +libsodium-static + +[debs] +libsodium-dev + +[download.not(os=windows)] +url = https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz +sha256 = 0cc3dae33e642cc187b5ceb467e0ad0e1b51dcba577de1190e9ffa17766ac2b1 + +[build.not(os=windows)] +builder = autoconf +subdir = libsodium-1.0.17 + +[download.os=windows] +url = https://download.libsodium.org/libsodium/releases/libsodium-1.0.17-msvc.zip +sha256 = f0f32ad8ebd76eee99bb039f843f583f2babca5288a8c26a7261db9694c11467 + +[build.os=windows] +builder = nop + +[install.files.os=windows] +x64/Release/v141/dynamic/libsodium.dll = bin/libsodium.dll +x64/Release/v141/dynamic/libsodium.lib = lib/libsodium.lib +x64/Release/v141/dynamic/libsodium.exp = lib/libsodium.exp +x64/Release/v141/dynamic/libsodium.pdb = lib/libsodium.pdb +include = include + +[autoconf.args] diff --git a/build/fbcode_builder/manifests/libtool b/build/fbcode_builder/manifests/libtool new file mode 100644 index 000000000000..1ec99b5f4512 --- /dev/null +++ b/build/fbcode_builder/manifests/libtool @@ -0,0 +1,22 @@ +[manifest] +name = libtool + +[rpms] +libtool + +[debs] +libtool + +[download] +url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz +sha256 = e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 + +[build] +builder = autoconf +subdir = libtool-2.4.6 + +[dependencies] +automake + +[autoconf.args] +--enable-ltdl-install diff --git a/build/fbcode_builder/manifests/libusb b/build/fbcode_builder/manifests/libusb new file mode 100644 index 000000000000..74702d3f0876 --- /dev/null +++ b/build/fbcode_builder/manifests/libusb @@ -0,0 +1,23 @@ +[manifest] +name = libusb + +[rpms] +libusb-devel +libusb + +[debs] +libusb-1.0-0-dev + +[download] +url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2 +sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157 + +[build.os=linux] +builder = autoconf +subdir = libusb-1.0.22 + +[autoconf.args] +# fboss (which added the libusb dep) doesn't need udev so it is disabled here. +# if someone in the future wants to add udev for something else, it won't hurt +# fboss. +--disable-udev diff --git a/build/fbcode_builder/manifests/libyaml b/build/fbcode_builder/manifests/libyaml new file mode 100644 index 000000000000..a7ff57316fe1 --- /dev/null +++ b/build/fbcode_builder/manifests/libyaml @@ -0,0 +1,13 @@ +[manifest] +name = libyaml + +[download] +url = http://pyyaml.org/download/libyaml/yaml-0.1.7.tar.gz +sha256 = 8088e457264a98ba451a90b8661fcb4f9d6f478f7265d48322a196cec2480729 + +[build.os=linux] +builder = autoconf +subdir = yaml-0.1.7 + +[build.not(os=linux)] +builder = nop diff --git a/build/fbcode_builder/manifests/libzmq b/build/fbcode_builder/manifests/libzmq new file mode 100644 index 000000000000..4f555fa65fcd --- /dev/null +++ b/build/fbcode_builder/manifests/libzmq @@ -0,0 +1,24 @@ +[manifest] +name = libzmq + +[rpms] +zeromq-devel +zeromq + +[debs] +libzmq3-dev + +[download] +url = https://github.com/zeromq/libzmq/releases/download/v4.3.1/zeromq-4.3.1.tar.gz +sha256 = bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb + + +[build] +builder = autoconf +subdir = zeromq-4.3.1 + +[autoconf.args] + +[dependencies] +autoconf +libtool diff --git a/build/fbcode_builder/manifests/lz4 b/build/fbcode_builder/manifests/lz4 new file mode 100644 index 000000000000..03dbd9de4046 --- /dev/null +++ b/build/fbcode_builder/manifests/lz4 @@ -0,0 +1,17 @@ +[manifest] +name = lz4 + +[rpms] +lz4-devel +lz4-static + +[debs] +liblz4-dev + +[download] +url = https://github.com/lz4/lz4/archive/v1.8.3.tar.gz +sha256 = 33af5936ac06536805f9745e0b6d61da606a1f8b4cc5c04dd3cbaca3b9b4fc43 + +[build] +builder = cmake +subdir = lz4-1.8.3/contrib/cmake_unofficial diff --git a/build/fbcode_builder/manifests/lzo b/build/fbcode_builder/manifests/lzo new file mode 100644 index 000000000000..342428ab5d2a --- /dev/null +++ b/build/fbcode_builder/manifests/lzo @@ -0,0 +1,19 @@ +[manifest] +name = lzo + +[rpms] +lzo-devel + +[debs] +liblzo2-dev + +[download] +url = http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz +sha256 = c0f892943208266f9b6543b3ae308fab6284c5c90e627931446fb49b4221a072 + +[build.not(os=windows)] +builder = autoconf +subdir = lzo-2.10 + +[build.os=windows] +builder = nop diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke new file mode 100644 index 000000000000..7df92c77b3bc --- /dev/null +++ b/build/fbcode_builder/manifests/mononoke @@ -0,0 +1,44 @@ +[manifest] +name = mononoke +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/eden.git + +[build.not(os=windows)] +builder = cargo + +[build.os=windows] +# building Mononoke on windows is not supported +builder = nop + +[cargo] +build_doc = true +workspace_dir = eden/mononoke + +[shipit.pathmap] +fbcode/configerator/structs/scm/mononoke/public_autocargo = configerator/structs/scm/mononoke +fbcode/configerator/structs/scm/mononoke = configerator/structs/scm/mononoke +fbcode/eden/oss = . +fbcode/eden = eden +fbcode/eden/mononoke/public_autocargo = eden/mononoke +fbcode/tools/lfs = tools/lfs +tools/rust/ossconfigs = . + +[shipit.strip] +# strip all code unrelated to mononoke to prevent triggering unnecessary checks +^fbcode/eden/(?!mononoke|scm/lib/xdiff.*)/.*$ +^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ +^fbcode/eden/mononoke/Cargo\.toml$ +^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$ +^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$ +^.*/facebook/.*$ + +[dependencies] +fbthrift-source +rust-shed + +[dependencies.fb=on] +rust diff --git a/build/fbcode_builder/manifests/mononoke_integration b/build/fbcode_builder/manifests/mononoke_integration new file mode 100644 index 000000000000..a796e967e6ae --- /dev/null +++ b/build/fbcode_builder/manifests/mononoke_integration @@ -0,0 +1,47 @@ +[manifest] +name = mononoke_integration +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[build.not(os=windows)] +builder = make +subdir = eden/mononoke/tests/integration + +[build.os=windows] +# building Mononoke on windows is not supported +builder = nop + +[make.build_args] +build-getdeps + +[make.install_args] +install-getdeps + +[make.test_args] +test-getdeps + +[shipit.pathmap] +fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration + +[shipit.strip] +^.*/facebook/.*$ + +[dependencies] +eden_scm +eden_scm_lib_edenapi_tools +jq +mononoke +nmap +python-click +python-dulwich +tree + +[dependencies.os=linux] +sqlite3-bin + +[dependencies.os=darwin] +gnu-bash +gnu-coreutils +gnu-grep +gnu-sed diff --git a/build/fbcode_builder/manifests/mvfst b/build/fbcode_builder/manifests/mvfst new file mode 100644 index 000000000000..4f72a9192458 --- /dev/null +++ b/build/fbcode_builder/manifests/mvfst @@ -0,0 +1,32 @@ +[manifest] +name = mvfst +fbsource_path = fbcode/quic +shipit_project = mvfst +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookincubator/mvfst.git + +[build] +builder = cmake +subdir = . + +[cmake.defines.test=on] +BUILD_TESTS = ON + +[cmake.defines.all(os=windows, test=on)] +BUILD_TESTS = OFF + +[cmake.defines.test=off] +BUILD_TESTS = OFF + +[dependencies] +folly +fizz + +[dependencies.all(test=on, not(os=windows))] +googletest_1_8 + +[shipit.pathmap] +fbcode/quic/public_root = . +fbcode/quic = quic diff --git a/build/fbcode_builder/manifests/nghttp2 b/build/fbcode_builder/manifests/nghttp2 new file mode 100644 index 000000000000..151daf8af700 --- /dev/null +++ b/build/fbcode_builder/manifests/nghttp2 @@ -0,0 +1,20 @@ +[manifest] +name = nghttp2 + +[rpms] +libnghttp2-devel +libnghttp2 + +[debs] +libnghttp2-dev + +[download] +url = https://github.com/nghttp2/nghttp2/releases/download/v1.39.2/nghttp2-1.39.2.tar.gz +sha256 = fc820a305e2f410fade1a3260f09229f15c0494fc089b0100312cd64a33a38c0 + +[build] +builder = autoconf +subdir = nghttp2-1.39.2 + +[autoconf.args] +--enable-lib-only diff --git a/build/fbcode_builder/manifests/ninja b/build/fbcode_builder/manifests/ninja new file mode 100644 index 000000000000..2b6c5dc8da70 --- /dev/null +++ b/build/fbcode_builder/manifests/ninja @@ -0,0 +1,26 @@ +[manifest] +name = ninja + +[rpms] +ninja-build + +[debs] +ninja-build + +[download.os=windows] +url = https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip +sha256 = bbde850d247d2737c5764c927d1071cbb1f1957dcabda4a130fa8547c12c695f + +[build.os=windows] +builder = nop + +[install.files.os=windows] +ninja.exe = bin/ninja.exe + +[download.not(os=windows)] +url = https://github.com/ninja-build/ninja/archive/v1.10.2.tar.gz +sha256 = ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed + +[build.not(os=windows)] +builder = ninja_bootstrap +subdir = ninja-1.10.2 diff --git a/build/fbcode_builder/manifests/nmap b/build/fbcode_builder/manifests/nmap new file mode 100644 index 000000000000..c245e12417be --- /dev/null +++ b/build/fbcode_builder/manifests/nmap @@ -0,0 +1,25 @@ +[manifest] +name = nmap + +[rpms] +nmap + +[debs] +nmap + +[download.not(os=windows)] +url = https://api.github.com/repos/nmap/nmap/tarball/ef8213a36c2e89233c806753a57b5cd473605408 +sha256 = eda39e5a8ef4964fac7db16abf91cc11ff568eac0fa2d680b0bfa33b0ed71f4a + +[build.not(os=windows)] +builder = autoconf +subdir = nmap-nmap-ef8213a +build_in_src_dir = true + +[build.os=windows] +builder = nop + +[autoconf.args] +# Without this option the build was filing to find some third party libraries +# that we don't need +enable_rdma=no diff --git a/build/fbcode_builder/manifests/openr b/build/fbcode_builder/manifests/openr new file mode 100644 index 000000000000..754ba8cd54b2 --- /dev/null +++ b/build/fbcode_builder/manifests/openr @@ -0,0 +1,37 @@ +[manifest] +name = openr +fbsource_path = facebook/openr +shipit_project = openr +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/openr.git + +[build.os=linux] +builder = cmake + +[build.not(os=linux)] +# boost.fiber is required and that is not available on macos. +# libzmq doesn't currently build on windows. +builder = nop + +[dependencies] +boost +fb303 +fbthrift +fbzmq +folly +googletest +re2 + +[cmake.defines.test=on] +BUILD_TESTS=ON +ADD_ROOT_TESTS=OFF + +[cmake.defines.test=off] +BUILD_TESTS=OFF + + +[shipit.pathmap] +fbcode/openr = openr +fbcode/openr/public_tld = . diff --git a/build/fbcode_builder/manifests/openssl b/build/fbcode_builder/manifests/openssl new file mode 100644 index 000000000000..991196c9a339 --- /dev/null +++ b/build/fbcode_builder/manifests/openssl @@ -0,0 +1,20 @@ +[manifest] +name = openssl + +[rpms] +openssl-devel +openssl + +[debs] +libssl-dev + +[download] +url = https://www.openssl.org/source/openssl-1.1.1i.tar.gz +sha256 = e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 + +[build] +builder = openssl +subdir = openssl-1.1.1i + +[dependencies.os=windows] +perl diff --git a/build/fbcode_builder/manifests/osxfuse b/build/fbcode_builder/manifests/osxfuse new file mode 100644 index 000000000000..b6c6c551f118 --- /dev/null +++ b/build/fbcode_builder/manifests/osxfuse @@ -0,0 +1,12 @@ +[manifest] +name = osxfuse + +[download] +url = https://github.com/osxfuse/osxfuse/archive/osxfuse-3.8.3.tar.gz +sha256 = 93bab6731bdfe8dc1ef069483437270ce7fe5a370f933d40d8d0ef09ba846c0c + +[build] +builder = nop + +[install.files] +osxfuse-osxfuse-3.8.3/common = include diff --git a/build/fbcode_builder/manifests/patchelf b/build/fbcode_builder/manifests/patchelf new file mode 100644 index 000000000000..f9d050424a29 --- /dev/null +++ b/build/fbcode_builder/manifests/patchelf @@ -0,0 +1,17 @@ +[manifest] +name = patchelf + +[rpms] +patchelf + +[debs] +patchelf + +[download] +url = https://github.com/NixOS/patchelf/archive/0.10.tar.gz +sha256 = b3cb6bdedcef5607ce34a350cf0b182eb979f8f7bc31eae55a93a70a3f020d13 + +[build] +builder = autoconf +subdir = patchelf-0.10 + diff --git a/build/fbcode_builder/manifests/pcre b/build/fbcode_builder/manifests/pcre new file mode 100644 index 000000000000..5353d8c27622 --- /dev/null +++ b/build/fbcode_builder/manifests/pcre @@ -0,0 +1,18 @@ +[manifest] +name = pcre + +[rpms] +pcre-devel +pcre-static + +[debs] +libpcre3-dev + +[download] +url = https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz +sha256 = 0b8e7465dc5e98c757cc3650a20a7843ee4c3edf50aaf60bb33fd879690d2c73 + +[build] +builder = cmake +subdir = pcre-8.43 + diff --git a/build/fbcode_builder/manifests/perl b/build/fbcode_builder/manifests/perl new file mode 100644 index 000000000000..32bddc51ca69 --- /dev/null +++ b/build/fbcode_builder/manifests/perl @@ -0,0 +1,11 @@ +[manifest] +name = perl + +[download.os=windows] +url = http://strawberryperl.com/download/5.28.1.1/strawberry-perl-5.28.1.1-64bit-portable.zip +sha256 = 935c95ba096fa11c4e1b5188732e3832d330a2a79e9882ab7ba8460ddbca810d + +[build.os=windows] +builder = nop +subdir = perl + diff --git a/build/fbcode_builder/manifests/pexpect b/build/fbcode_builder/manifests/pexpect new file mode 100644 index 000000000000..682e66a540c1 --- /dev/null +++ b/build/fbcode_builder/manifests/pexpect @@ -0,0 +1,12 @@ +[manifest] +name = pexpect + +[download] +url = https://files.pythonhosted.org/packages/0e/3e/377007e3f36ec42f1b84ec322ee12141a9e10d808312e5738f52f80a232c/pexpect-4.7.0-py2.py3-none-any.whl +sha256 = 2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1 + +[build] +builder = python-wheel + +[dependencies] +python-ptyprocess diff --git a/build/fbcode_builder/manifests/protobuf b/build/fbcode_builder/manifests/protobuf new file mode 100644 index 000000000000..7f21e4821741 --- /dev/null +++ b/build/fbcode_builder/manifests/protobuf @@ -0,0 +1,17 @@ +[manifest] +name = protobuf + +[rpms] +protobuf-devel + +[debs] +libprotobuf-dev + +[git] +repo_url = https://github.com/protocolbuffers/protobuf.git + +[build.not(os=windows)] +builder = autoconf + +[build.os=windows] +builder = nop diff --git a/build/fbcode_builder/manifests/proxygen b/build/fbcode_builder/manifests/proxygen new file mode 100644 index 000000000000..5452a24544ca --- /dev/null +++ b/build/fbcode_builder/manifests/proxygen @@ -0,0 +1,39 @@ +[manifest] +name = proxygen +fbsource_path = fbcode/proxygen +shipit_project = proxygen +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/proxygen.git + +[build.os=windows] +builder = nop + +[build] +builder = cmake +subdir = . + +[cmake.defines] +BUILD_QUIC = ON + +[cmake.defines.test=on] +BUILD_TESTS = ON + +[cmake.defines.test=off] +BUILD_TESTS = OFF + +[dependencies] +zlib +gperf +folly +fizz +wangle +mvfst + +[dependencies.test=on] +googletest_1_8 + +[shipit.pathmap] +fbcode/proxygen/public_tld = . +fbcode/proxygen = proxygen diff --git a/build/fbcode_builder/manifests/python b/build/fbcode_builder/manifests/python new file mode 100644 index 000000000000..e51c0ab510bc --- /dev/null +++ b/build/fbcode_builder/manifests/python @@ -0,0 +1,17 @@ +[manifest] +name = python + +[rpms] +python3 +python3-devel + +[debs] +python3-all-dev + +[download.os=linux] +url = https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tgz +sha256 = aeee681c235ad336af116f08ab6563361a0c81c537072c1b309d6e4050aa2114 + +[build.os=linux] +builder = autoconf +subdir = Python-3.7.6 diff --git a/build/fbcode_builder/manifests/python-click b/build/fbcode_builder/manifests/python-click new file mode 100644 index 000000000000..ea9a9d2d3dc3 --- /dev/null +++ b/build/fbcode_builder/manifests/python-click @@ -0,0 +1,9 @@ +[manifest] +name = python-click + +[download] +url = https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl +sha256 = dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc + +[build] +builder = python-wheel diff --git a/build/fbcode_builder/manifests/python-dulwich b/build/fbcode_builder/manifests/python-dulwich new file mode 100644 index 000000000000..0d995e12f4b3 --- /dev/null +++ b/build/fbcode_builder/manifests/python-dulwich @@ -0,0 +1,19 @@ +[manifest] +name = python-dulwich + +# The below links point to custom github forks of project dulwich, because the +# 0.18.6 version didn't have an official rollout of wheel packages. + +[download.os=linux] +url = https://github.com/lukaspiatkowski/dulwich/releases/download/dulwich-0.18.6-wheel/dulwich-0.18.6-cp36-cp36m-linux_x86_64.whl +sha256 = e96f545f3d003e67236785473caaba2c368e531ea85fd508a3bd016ebac3a6d8 + +[download.os=darwin] +url = https://github.com/lukaspiatkowski/dulwich/releases/download/dulwich-0.18.6-wheel/dulwich-0.18.6-cp37-cp37m-macosx_10_14_x86_64.whl +sha256 = 8373652056284ad40ea5220b659b3489b0a91f25536322345a3e4b5d29069308 + +[build.not(os=windows)] +builder = python-wheel + +[build.os=windows] +builder = nop diff --git a/build/fbcode_builder/manifests/python-ptyprocess b/build/fbcode_builder/manifests/python-ptyprocess new file mode 100644 index 000000000000..adc60e048ed1 --- /dev/null +++ b/build/fbcode_builder/manifests/python-ptyprocess @@ -0,0 +1,9 @@ +[manifest] +name = python-ptyprocess + +[download] +url = https://files.pythonhosted.org/packages/d1/29/605c2cc68a9992d18dada28206eeada56ea4bd07a239669da41674648b6f/ptyprocess-0.6.0-py2.py3-none-any.whl +sha256 = d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f + +[build] +builder = python-wheel diff --git a/build/fbcode_builder/manifests/python-six b/build/fbcode_builder/manifests/python-six new file mode 100644 index 000000000000..a712188dc290 --- /dev/null +++ b/build/fbcode_builder/manifests/python-six @@ -0,0 +1,9 @@ +[manifest] +name = python-six + +[download] +url = https://files.pythonhosted.org/packages/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl +sha256 = 3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c + +[build] +builder = python-wheel diff --git a/build/fbcode_builder/manifests/python-toml b/build/fbcode_builder/manifests/python-toml new file mode 100644 index 000000000000..b49a3b8fb860 --- /dev/null +++ b/build/fbcode_builder/manifests/python-toml @@ -0,0 +1,9 @@ +[manifest] +name = python-toml + +[download] +url = https://files.pythonhosted.org/packages/a2/12/ced7105d2de62fa7c8fb5fce92cc4ce66b57c95fb875e9318dba7f8c5db0/toml-0.10.0-py2.py3-none-any.whl +sha256 = 235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e + +[build] +builder = python-wheel diff --git a/build/fbcode_builder/manifests/re2 b/build/fbcode_builder/manifests/re2 new file mode 100644 index 000000000000..eb4d6a92cb8f --- /dev/null +++ b/build/fbcode_builder/manifests/re2 @@ -0,0 +1,17 @@ +[manifest] +name = re2 + +[rpms] +re2 +re2-devel + +[debs] +libre2-dev + +[download] +url = https://github.com/google/re2/archive/2019-06-01.tar.gz +sha256 = 02b7d73126bd18e9fbfe5d6375a8bb13fadaf8e99e48cbb062e4500fc18e8e2e + +[build] +builder = cmake +subdir = re2-2019-06-01 diff --git a/build/fbcode_builder/manifests/rocksdb b/build/fbcode_builder/manifests/rocksdb new file mode 100644 index 000000000000..323e6dc6d831 --- /dev/null +++ b/build/fbcode_builder/manifests/rocksdb @@ -0,0 +1,41 @@ +[manifest] +name = rocksdb + +[download] +url = https://github.com/facebook/rocksdb/archive/v6.8.1.tar.gz +sha256 = ca192a06ed3bcb9f09060add7e9d0daee1ae7a8705a3d5ecbe41867c5e2796a2 + +[dependencies] +lz4 +snappy + +[build] +builder = cmake +subdir = rocksdb-6.8.1 + +[cmake.defines] +WITH_SNAPPY=ON +WITH_LZ4=ON +WITH_TESTS=OFF +WITH_BENCHMARK_TOOLS=OFF +# We get relocation errors with the static gflags lib, +# and there's no clear way to make it pick the shared gflags +# so just turn it off. +WITH_GFLAGS=OFF +# mac pro machines don't have some of the newer features that +# rocksdb enables by default; ask it to disable their use even +# when building on new hardware +PORTABLE = ON +# Disable the use of -Werror +FAIL_ON_WARNINGS = OFF + +[cmake.defines.os=windows] +ROCKSDB_INSTALL_ON_WINDOWS=ON +# RocksDB hard codes the paths to the snappy libs to something +# that doesn't exist; ignoring the usual cmake rules. As a result, +# we can't build it with snappy without either patching rocksdb or +# without introducing more complex logic to the build system to +# connect the snappy build outputs to rocksdb's custom logic here. +# Let's just turn it off on windows. +WITH_SNAPPY=OFF +WITH_LZ4=OFF diff --git a/build/fbcode_builder/manifests/rust-shed b/build/fbcode_builder/manifests/rust-shed new file mode 100644 index 000000000000..c94b3fdd60f1 --- /dev/null +++ b/build/fbcode_builder/manifests/rust-shed @@ -0,0 +1,34 @@ +[manifest] +name = rust-shed +fbsource_path = fbcode/common/rust/shed +shipit_project = rust-shed +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/rust-shed.git + +[build] +builder = cargo + +[cargo] +build_doc = true +workspace_dir = + +[shipit.pathmap] +fbcode/common/rust/shed = shed +fbcode/common/rust/shed/public_autocargo = shed +fbcode/common/rust/shed/public_tld = . +tools/rust/ossconfigs = . + +[shipit.strip] +^fbcode/common/rust/shed/(?!public_autocargo|public_tld).+/Cargo\.toml$ + +[dependencies] +fbthrift +# macOS doesn't expose the openssl api so we need to build our own. +# Windows doesn't have openssl and Linux might contain an old version, +# so we get to provide it +openssl + +[dependencies.fb=on] +rust diff --git a/build/fbcode_builder/manifests/snappy b/build/fbcode_builder/manifests/snappy new file mode 100644 index 000000000000..2f46a7734bd3 --- /dev/null +++ b/build/fbcode_builder/manifests/snappy @@ -0,0 +1,25 @@ +[manifest] +name = snappy + +[rpms] +snappy +snappy-devel + +[debs] +libsnappy-dev + +[download] +url = https://github.com/google/snappy/archive/1.1.7.tar.gz +sha256 = 3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4 + +[build] +builder = cmake +subdir = snappy-1.1.7 + +[cmake.defines] +SNAPPY_BUILD_TESTS = OFF + +# Avoid problems like `relocation R_X86_64_PC32 against symbol` on ELF systems +# when linking rocksdb, which builds PIC even when building a static lib +[cmake.defines.os=linux] +BUILD_SHARED_LIBS = ON diff --git a/build/fbcode_builder/manifests/sqlite3 b/build/fbcode_builder/manifests/sqlite3 new file mode 100644 index 000000000000..2463f576178b --- /dev/null +++ b/build/fbcode_builder/manifests/sqlite3 @@ -0,0 +1,21 @@ +[manifest] +name = sqlite3 + +[rpms] +sqlite-devel +sqlite-libs + +[debs] +libsqlite3-dev + +[download] +url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip +sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327 + +[dependencies] +cmake +ninja + +[build] +builder = sqlite +subdir = sqlite-amalgamation-3280000 diff --git a/build/fbcode_builder/manifests/sqlite3-bin b/build/fbcode_builder/manifests/sqlite3-bin new file mode 100644 index 000000000000..aa138d499d6b --- /dev/null +++ b/build/fbcode_builder/manifests/sqlite3-bin @@ -0,0 +1,28 @@ +[manifest] +name = sqlite3-bin + +[rpms] +sqlite + +[debs] +sqlite3 + +[download.os=linux] +url = https://github.com/sqlite/sqlite/archive/version-3.33.0.tar.gz +sha256 = 48e5f989eefe9af0ac758096f82ead0f3c7b58118ac17cc5810495bd5084a331 + +[build.os=linux] +builder = autoconf +subdir = sqlite-version-3.33.0 + +[build.not(os=linux)] +# MacOS comes with sqlite3 preinstalled and don't need Windows here +builder = nop + +[dependencies.os=linux] +tcl + +[autoconf.args] +# This flag disabled tcl as a runtime library used for some functionality, +# but tcl is still a required dependency as it is used by the build files +--disable-tcl diff --git a/build/fbcode_builder/manifests/tcl b/build/fbcode_builder/manifests/tcl new file mode 100644 index 000000000000..5e9892f37a6d --- /dev/null +++ b/build/fbcode_builder/manifests/tcl @@ -0,0 +1,20 @@ +[manifest] +name = tcl + +[rpms] +tcl + +[debs] +tcl + +[download] +url = https://github.com/tcltk/tcl/archive/core-8-7a3.tar.gz +sha256 = 22d748f0c9652f3ecc195fed3f24a1b6eea8d449003085e6651197951528982e + +[build.os=linux] +builder = autoconf +subdir = tcl-core-8-7a3/unix + +[build.not(os=linux)] +# This is for sqlite3 on Linux for now +builder = nop diff --git a/build/fbcode_builder/manifests/tree b/build/fbcode_builder/manifests/tree new file mode 100644 index 000000000000..0c982f35a773 --- /dev/null +++ b/build/fbcode_builder/manifests/tree @@ -0,0 +1,34 @@ +[manifest] +name = tree + +[rpms] +tree + +[debs] +tree + +[download.os=linux] +url = https://salsa.debian.org/debian/tree-packaging/-/archive/debian/1.8.0-1/tree-packaging-debian-1.8.0-1.tar.gz +sha256 = a841eee1d52bfd64a48f54caab9937b9bd92935055c48885c4ab1ae4dab7fae5 + +[download.os=darwin] +# The official package of tree source requires users of non-Linux platform to +# comment/uncomment certain lines in the Makefile to build for their platform. +# Besauce getdeps.py doesn't have that functionality we just use this custom +# fork of tree which has proper lines uncommented for a OSX build +url = https://github.com/lukaspiatkowski/tree-command/archive/debian/1.8.0-1-macos.tar.gz +sha256 = 9cbe889553d95cf5a2791dd0743795d46a3c092c5bba691769c0e5c52e11229e + +[build.os=linux] +builder = make +subdir = tree-packaging-debian-1.8.0-1 + +[build.os=darwin] +builder = make +subdir = tree-command-debian-1.8.0-1-macos + +[build.os=windows] +builder = nop + +[make.install_args] +install diff --git a/build/fbcode_builder/manifests/wangle b/build/fbcode_builder/manifests/wangle new file mode 100644 index 000000000000..6b330d620f46 --- /dev/null +++ b/build/fbcode_builder/manifests/wangle @@ -0,0 +1,27 @@ +[manifest] +name = wangle +fbsource_path = fbcode/wangle +shipit_project = wangle +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/wangle.git + +[build] +builder = cmake +subdir = wangle + +[cmake.defines.test=on] +BUILD_TESTS=ON + +[cmake.defines.test=off] +BUILD_TESTS=OFF + +[dependencies] +folly +googletest +fizz + +[shipit.pathmap] +fbcode/wangle/public_tld = . +fbcode/wangle = wangle diff --git a/build/fbcode_builder/manifests/watchman b/build/fbcode_builder/manifests/watchman new file mode 100644 index 000000000000..0fcd6bb9f073 --- /dev/null +++ b/build/fbcode_builder/manifests/watchman @@ -0,0 +1,45 @@ +[manifest] +name = watchman +fbsource_path = fbcode/watchman +shipit_project = watchman +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/watchman.git + +[build] +builder = cmake + +[dependencies] +boost +cpptoml +fb303 +fbthrift +folly +pcre +googletest + +[dependencies.fb=on] +rust + +[shipit.pathmap] +fbcode/watchman = watchman +fbcode/watchman/oss = . +fbcode/eden/fs = eden/fs + +[shipit.strip] +^fbcode/eden/fs/(?!.*\.thrift|service/shipit_test_file\.txt) + +[cmake.defines.fb=on] +ENABLE_EDEN_SUPPORT=ON + +# FB macos specific settings +[cmake.defines.all(fb=on,os=darwin)] +# this path is coupled with the FB internal watchman-osx.spec +WATCHMAN_STATE_DIR=/opt/facebook/watchman/var/run/watchman +# tell cmake not to try to create /opt/facebook/... +INSTALL_WATCHMAN_STATE_DIR=OFF +USE_SYS_PYTHON=OFF + +[depends.environment] +WATCHMAN_VERSION_OVERRIDE diff --git a/build/fbcode_builder/manifests/yaml-cpp b/build/fbcode_builder/manifests/yaml-cpp new file mode 100644 index 000000000000..bffa540fe78e --- /dev/null +++ b/build/fbcode_builder/manifests/yaml-cpp @@ -0,0 +1,20 @@ +[manifest] +name = yaml-cpp + +[download] +url = https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.6.2.tar.gz +sha256 = e4d8560e163c3d875fd5d9e5542b5fd5bec810febdcba61481fe5fc4e6b1fd05 + +[build.os=linux] +builder = cmake +subdir = yaml-cpp-yaml-cpp-0.6.2 + +[build.not(os=linux)] +builder = nop + +[dependencies] +boost +googletest + +[cmake.defines] +YAML_CPP_BUILD_TESTS=OFF diff --git a/build/fbcode_builder/manifests/zlib b/build/fbcode_builder/manifests/zlib new file mode 100644 index 000000000000..8df0e3e48b37 --- /dev/null +++ b/build/fbcode_builder/manifests/zlib @@ -0,0 +1,22 @@ +[manifest] +name = zlib + +[rpms] +zlib-devel +zlib-static + +[debs] +zlib1g-dev + +[download] +url = http://www.zlib.net/zlib-1.2.11.tar.gz +sha256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 + +[build.os=windows] +builder = cmake +subdir = zlib-1.2.11 + +# Every platform but windows ships with zlib, so just skip +# building on not(windows) +[build.not(os=windows)] +builder = nop diff --git a/build/fbcode_builder/manifests/zstd b/build/fbcode_builder/manifests/zstd new file mode 100644 index 000000000000..71db9d5c6e56 --- /dev/null +++ b/build/fbcode_builder/manifests/zstd @@ -0,0 +1,28 @@ +[manifest] +name = zstd + +[rpms] +libzstd-devel +libzstd + +[debs] +libzstd-dev + +[download] +url = https://github.com/facebook/zstd/releases/download/v1.4.5/zstd-1.4.5.tar.gz +sha256 = 98e91c7c6bf162bf90e4e70fdbc41a8188b9fa8de5ad840c401198014406ce9e + +[build] +builder = cmake +subdir = zstd-1.4.5/build/cmake + +# The zstd cmake build explicitly sets the install name +# for the shared library in such a way that cmake discards +# the path to the library from the install_name, rendering +# the library non-resolvable during the build. The short +# term solution for this is just to link static on macos. +[cmake.defines.os=darwin] +ZSTD_BUILD_SHARED = OFF + +[cmake.defines.os=windows] +ZSTD_BUILD_SHARED = OFF diff --git a/build/fbcode_builder/parse_args.py b/build/fbcode_builder/parse_args.py new file mode 100644 index 000000000000..8d5e353308a0 --- /dev/null +++ b/build/fbcode_builder/parse_args.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +"Argument parsing logic shared by all fbcode_builder CLI tools." + +import argparse +import logging + +from shell_quoting import raw_shell, ShellQuoted + + +def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help): + """ + + Provides some standard arguments: --debug, --option, --shell-quoted-option + + Then, calls `add_args_fn(parser)` to add application-specific arguments. + + `opts` are first used as defaults for the various command-line + arguments. Then, the parsed arguments are mapped back into `opts`, + which then become the values for `FBCodeBuilder.option()`, to be used + both by the builder and by `get_steps_fn()`. + + `help` is printed in response to the `--help` argument. + + """ + top_level_opts = set(top_level_opts) + + parser = argparse.ArgumentParser( + description=help, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + add_args_fn(parser) + + parser.add_argument( + "--option", + nargs=2, + metavar=("KEY", "VALUE"), + action="append", + default=[ + (k, v) + for k, v in opts.items() + if k not in top_level_opts and not isinstance(v, ShellQuoted) + ], + help="Set project-specific options. These are assumed to be raw " + "strings, to be shell-escaped as needed. Default: %(default)s.", + ) + parser.add_argument( + "--shell-quoted-option", + nargs=2, + metavar=("KEY", "VALUE"), + action="append", + default=[ + (k, raw_shell(v)) + for k, v in opts.items() + if k not in top_level_opts and isinstance(v, ShellQuoted) + ], + help="Set project-specific options. These are assumed to be shell-" + "quoted, and may be used in commands as-is. Default: %(default)s.", + ) + + parser.add_argument("--debug", action="store_true", help="Log more") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + # Map command-line args back into opts. + logging.debug("opts before command-line arguments: {0}".format(opts)) + + new_opts = {} + for key in top_level_opts: + val = getattr(args, key) + # Allow clients to unset a default by passing a value of None in opts + if val is not None: + new_opts[key] = val + for key, val in args.option: + new_opts[key] = val + for key, val in args.shell_quoted_option: + new_opts[key] = ShellQuoted(val) + + logging.debug("opts after command-line arguments: {0}".format(new_opts)) + + return new_opts diff --git a/build/fbcode_builder/shell_builder.py b/build/fbcode_builder/shell_builder.py new file mode 100644 index 000000000000..e0d5429ad42b --- /dev/null +++ b/build/fbcode_builder/shell_builder.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +""" +shell_builder.py allows running the fbcode_builder logic +on the host rather than in a container. + +It emits a bash script with set -exo pipefail configured such that +any failing step will cause the script to exit with failure. + +== How to run it? == + +cd build +python fbcode_builder/shell_builder.py > ~/run.sh +bash ~/run.sh +""" + +import distutils.spawn +import os + +from fbcode_builder import FBCodeBuilder +from shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted +from utils import recursively_flatten_list + + +class ShellFBCodeBuilder(FBCodeBuilder): + def _render_impl(self, steps): + return raw_shell(shell_join("\n", recursively_flatten_list(steps))) + + def set_env(self, key, value): + return ShellQuoted("export {key}={val}").format(key=key, val=value) + + def workdir(self, dir): + return [ + ShellQuoted("mkdir -p {d} && cd {d}").format(d=dir), + ] + + def run(self, shell_cmd): + return ShellQuoted("{cmd}").format(cmd=shell_cmd) + + def step(self, name, actions): + assert "\n" not in name, "Name {0} would span > 1 line".format(name) + b = ShellQuoted("") + return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b] + + def setup(self): + steps = ( + [ + ShellQuoted("set -exo pipefail"), + ] + + self.create_python_venv() + + self.python_venv() + ) + if self.has_option("ccache_dir"): + ccache_dir = self.option("ccache_dir") + steps += [ + ShellQuoted( + # Set CCACHE_DIR before the `ccache` invocations below. + "export CCACHE_DIR={ccache_dir} " + 'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"' + ).format(ccache_dir=ccache_dir) + ] + return steps + + def comment(self, comment): + return shell_comment(comment) + + def copy_local_repo(self, dir, dest_name): + return [ + ShellQuoted("cp -r {dir} {dest_name}").format(dir=dir, dest_name=dest_name), + ] + + +def find_project_root(): + here = os.path.dirname(os.path.realpath(__file__)) + maybe_root = os.path.dirname(os.path.dirname(here)) + if os.path.isdir(os.path.join(maybe_root, ".git")): + return maybe_root + raise RuntimeError( + "I expected shell_builder.py to be in the " + "build/fbcode_builder subdir of a git repo" + ) + + +def persistent_temp_dir(repo_root): + escaped = repo_root.replace("/", "sZs").replace("\\", "sZs").replace(":", "") + return os.path.join(os.path.expandvars("$HOME"), ".fbcode_builder-" + escaped) + + +if __name__ == "__main__": + from utils import read_fbcode_builder_config, build_fbcode_builder_config + + repo_root = find_project_root() + temp = persistent_temp_dir(repo_root) + + config = read_fbcode_builder_config("fbcode_builder_config.py") + builder = ShellFBCodeBuilder(projects_dir=temp) + + if distutils.spawn.find_executable("ccache"): + builder.add_option( + "ccache_dir", os.environ.get("CCACHE_DIR", os.path.join(temp, ".ccache")) + ) + builder.add_option("prefix", os.path.join(temp, "installed")) + builder.add_option("make_parallelism", 4) + builder.add_option( + "{project}:local_repo_dir".format(project=config["github_project"]), repo_root + ) + make_steps = build_fbcode_builder_config(config) + steps = make_steps(builder) + print(builder.render(steps)) diff --git a/build/fbcode_builder/shell_quoting.py b/build/fbcode_builder/shell_quoting.py new file mode 100644 index 000000000000..7429226bddb4 --- /dev/null +++ b/build/fbcode_builder/shell_quoting.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +""" + +Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping +too little or too much tends to be the most common error. The utilities in +this file give a systematic way of avoiding such bugs: + - When you write literal strings destined for the shell, use `ShellQuoted`. + - When these literal strings are parameterized, use `ShellQuoted.format`. + - Any parameters that are raw strings get `shell_quote`d automatically, + while any ShellQuoted parameters will be left intact. + - Use `path_join` to join path components. + - Use `shell_join` to join already-quoted command arguments or shell lines. + +""" + +import os +from collections import namedtuple + + +class ShellQuoted(namedtuple("ShellQuoted", ("do_not_use_raw_str",))): + """ + + Wrap a string with this to make it transparent to shell_quote(). It + will almost always suffice to use ShellQuoted.format(), path_join(), + or shell_join(). + + If you really must, use raw_shell() to access the raw string. + + """ + + def __new__(cls, s): + "No need to nest ShellQuoted." + return super(ShellQuoted, cls).__new__( + cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s + ) + + def __str__(self): + raise RuntimeError( + "One does not simply convert {0} to a string -- use path_join() " + "or ShellQuoted.format() instead".format(repr(self)) + ) + + def __repr__(self): + return "{0}({1})".format(self.__class__.__name__, repr(self.do_not_use_raw_str)) + + def format(self, **kwargs): + """ + + Use instead of str.format() when the arguments are either + `ShellQuoted()` or raw strings needing to be `shell_quote()`d. + + Positional args are deliberately not supported since they are more + error-prone. + + """ + return ShellQuoted( + self.do_not_use_raw_str.format( + **dict( + (k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items() + ) + ) + ) + + +def shell_quote(s): + "Quotes a string if it is not already quoted" + return ( + s + if isinstance(s, ShellQuoted) + else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'") + ) + + +def raw_shell(s): + "Not a member of ShellQuoted so we get a useful error for raw strings" + if isinstance(s, ShellQuoted): + return s.do_not_use_raw_str + raise RuntimeError("{0} should have been ShellQuoted".format(s)) + + +def shell_join(delim, it): + "Joins an iterable of ShellQuoted with a delimiter between each two" + return ShellQuoted(delim.join(raw_shell(s) for s in it)) + + +def path_join(*args): + "Joins ShellQuoted and raw pieces of paths to make a shell-quoted path" + return ShellQuoted(os.path.join(*[raw_shell(shell_quote(s)) for s in args])) + + +def shell_comment(c): + "Do not shell-escape raw strings in comments, but do handle line breaks." + return ShellQuoted("# {c}").format( + c=ShellQuoted( + (raw_shell(c) if isinstance(c, ShellQuoted) else c).replace("\n", "\n# ") + ) + ) diff --git a/build/fbcode_builder/specs/__init__.py b/build/fbcode_builder/specs/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build/fbcode_builder/specs/fbthrift.py b/build/fbcode_builder/specs/fbthrift.py new file mode 100644 index 000000000000..f0c7e7ac7c7f --- /dev/null +++ b/build/fbcode_builder/specs/fbthrift.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fizz as fizz +import specs.fmt as fmt +import specs.folly as folly +import specs.sodium as sodium +import specs.wangle as wangle +import specs.zstd as zstd + + +def fbcode_builder_spec(builder): + return { + "depends_on": [fmt, folly, fizz, sodium, wangle, zstd], + "steps": [ + builder.fb_github_cmake_install("fbthrift/thrift"), + ], + } diff --git a/build/fbcode_builder/specs/fbzmq.py b/build/fbcode_builder/specs/fbzmq.py new file mode 100644 index 000000000000..78c8bc9dd97c --- /dev/null +++ b/build/fbcode_builder/specs/fbzmq.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fbthrift as fbthrift +import specs.fmt as fmt +import specs.folly as folly +import specs.gmock as gmock +import specs.sodium as sodium +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option("zeromq/libzmq:git_hash", "v4.2.2") + return { + "depends_on": [fmt, folly, fbthrift, gmock, sodium], + "steps": [ + builder.github_project_workdir("zeromq/libzmq", "."), + builder.step( + "Build and install zeromq/libzmq", + [ + builder.run(ShellQuoted("./autogen.sh")), + builder.configure(), + builder.make_and_install(), + ], + ), + builder.fb_github_project_workdir("fbzmq/_build", "facebook"), + builder.step( + "Build and install fbzmq/", + [ + builder.cmake_configure("fbzmq/_build"), + # we need the pythonpath to find the thrift compiler + builder.run( + ShellQuoted( + 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' + "make -j {n}" + ).format( + p=builder.option("prefix"), + n=builder.option("make_parallelism"), + ) + ), + builder.run(ShellQuoted("make install")), + ], + ), + ], + } diff --git a/build/fbcode_builder/specs/fizz.py b/build/fbcode_builder/specs/fizz.py new file mode 100644 index 000000000000..82f26e67c4f0 --- /dev/null +++ b/build/fbcode_builder/specs/fizz.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fmt as fmt +import specs.folly as folly +import specs.gmock as gmock +import specs.sodium as sodium +import specs.zstd as zstd + + +def fbcode_builder_spec(builder): + builder.add_option( + "fizz/fizz/build:cmake_defines", + { + # Fizz's build is kind of broken, in the sense that both `mvfst` + # and `proxygen` depend on files that are only installed with + # `BUILD_TESTS` enabled, e.g. `fizz/crypto/test/TestUtil.h`. + "BUILD_TESTS": "ON" + }, + ) + return { + "depends_on": [gmock, fmt, folly, sodium, zstd], + "steps": [ + builder.fb_github_cmake_install( + "fizz/fizz/build", github_org="facebookincubator" + ) + ], + } diff --git a/build/fbcode_builder/specs/fmt.py b/build/fbcode_builder/specs/fmt.py new file mode 100644 index 000000000000..3953167995b7 --- /dev/null +++ b/build/fbcode_builder/specs/fmt.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option("fmtlib/fmt:git_hash", "6.2.1") + builder.add_option( + "fmtlib/fmt:cmake_defines", + { + # Avoids a bizarred failure to run tests in Bistro: + # test_crontab_selector: error while loading shared libraries: + # libfmt.so.6: cannot open shared object file: + # No such file or directory + "BUILD_SHARED_LIBS": "OFF", + }, + ) + return { + "steps": [ + builder.github_project_workdir("fmtlib/fmt", "build"), + builder.cmake_install("fmtlib/fmt"), + ], + } diff --git a/build/fbcode_builder/specs/folly.py b/build/fbcode_builder/specs/folly.py new file mode 100644 index 000000000000..e89d5e955e1c --- /dev/null +++ b/build/fbcode_builder/specs/folly.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fmt as fmt + + +def fbcode_builder_spec(builder): + return { + "depends_on": [fmt], + "steps": [ + # on macOS the filesystem is typically case insensitive. + # We need to ensure that the CWD is not the folly source + # dir when we build, otherwise the system will decide + # that `folly/String.h` is the file it wants when including + # `string.h` and the build will fail. + builder.fb_github_project_workdir("folly/_build"), + builder.cmake_install("facebook/folly"), + ], + } diff --git a/build/fbcode_builder/specs/gmock.py b/build/fbcode_builder/specs/gmock.py new file mode 100644 index 000000000000..774137301c26 --- /dev/null +++ b/build/fbcode_builder/specs/gmock.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option("google/googletest:git_hash", "release-1.8.1") + builder.add_option( + "google/googletest:cmake_defines", + { + "BUILD_GTEST": "ON", + # Avoid problems with MACOSX_RPATH + "BUILD_SHARED_LIBS": "OFF", + }, + ) + return { + "steps": [ + builder.github_project_workdir("google/googletest", "build"), + builder.cmake_install("google/googletest"), + ], + } diff --git a/build/fbcode_builder/specs/mvfst.py b/build/fbcode_builder/specs/mvfst.py new file mode 100644 index 000000000000..ce8b003d9a91 --- /dev/null +++ b/build/fbcode_builder/specs/mvfst.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fizz as fizz +import specs.folly as folly +import specs.gmock as gmock + + +def fbcode_builder_spec(builder): + # Projects that **depend** on mvfst should don't need to build tests. + builder.add_option( + "mvfst/build:cmake_defines", + { + # This is set to ON in the mvfst `fbcode_builder_config.py` + "BUILD_TESTS": "OFF" + }, + ) + return { + "depends_on": [gmock, folly, fizz], + "steps": [ + builder.fb_github_cmake_install( + "mvfst/build", github_org="facebookincubator" + ) + ], + } diff --git a/build/fbcode_builder/specs/proxygen.py b/build/fbcode_builder/specs/proxygen.py new file mode 100644 index 000000000000..6a584d71081f --- /dev/null +++ b/build/fbcode_builder/specs/proxygen.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fizz as fizz +import specs.fmt as fmt +import specs.folly as folly +import specs.gmock as gmock +import specs.mvfst as mvfst +import specs.sodium as sodium +import specs.wangle as wangle +import specs.zstd as zstd + + +def fbcode_builder_spec(builder): + # Projects that **depend** on proxygen should don't need to build tests + # or QUIC support. + builder.add_option( + "proxygen/proxygen:cmake_defines", + { + # These 2 are set to ON in `proxygen_quic.py` + "BUILD_QUIC": "OFF", + "BUILD_TESTS": "OFF", + # For bistro + "BUILD_SHARED_LIBS": "OFF", + }, + ) + + return { + "depends_on": [gmock, fmt, folly, wangle, fizz, sodium, zstd, mvfst], + "steps": [builder.fb_github_cmake_install("proxygen/proxygen", "..")], + } diff --git a/build/fbcode_builder/specs/proxygen_quic.py b/build/fbcode_builder/specs/proxygen_quic.py new file mode 100644 index 000000000000..b4959fb89c90 --- /dev/null +++ b/build/fbcode_builder/specs/proxygen_quic.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fizz as fizz +import specs.fmt as fmt +import specs.folly as folly +import specs.gmock as gmock +import specs.mvfst as mvfst +import specs.sodium as sodium +import specs.wangle as wangle +import specs.zstd as zstd + +# DO NOT USE THIS AS A LIBRARY -- this is currently effectively just part +# ofthe implementation of proxygen's `fbcode_builder_config.py`. This is +# why this builds tests and sets `BUILD_QUIC`. +def fbcode_builder_spec(builder): + builder.add_option( + "proxygen/proxygen:cmake_defines", + {"BUILD_QUIC": "ON", "BUILD_SHARED_LIBS": "OFF", "BUILD_TESTS": "ON"}, + ) + return { + "depends_on": [gmock, fmt, folly, wangle, fizz, sodium, zstd, mvfst], + "steps": [builder.fb_github_cmake_install("proxygen/proxygen", "..")], + } diff --git a/build/fbcode_builder/specs/re2.py b/build/fbcode_builder/specs/re2.py new file mode 100644 index 000000000000..cf4e08a0bd9c --- /dev/null +++ b/build/fbcode_builder/specs/re2.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + return { + "steps": [ + builder.github_project_workdir("google/re2", "build"), + builder.cmake_install("google/re2"), + ], + } diff --git a/build/fbcode_builder/specs/rocksdb.py b/build/fbcode_builder/specs/rocksdb.py new file mode 100644 index 000000000000..9ebfe4739424 --- /dev/null +++ b/build/fbcode_builder/specs/rocksdb.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option( + "rocksdb/_build:cmake_defines", + { + "USE_RTTI": "1", + "PORTABLE": "ON", + }, + ) + return { + "steps": [ + builder.fb_github_cmake_install("rocksdb/_build"), + ], + } diff --git a/build/fbcode_builder/specs/sodium.py b/build/fbcode_builder/specs/sodium.py new file mode 100644 index 000000000000..8be9833cfe4e --- /dev/null +++ b/build/fbcode_builder/specs/sodium.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option("jedisct1/libsodium:git_hash", "stable") + return { + "steps": [ + builder.github_project_workdir("jedisct1/libsodium", "."), + builder.step( + "Build and install jedisct1/libsodium", + [ + builder.run(ShellQuoted("./autogen.sh")), + builder.configure(), + builder.make_and_install(), + ], + ), + ], + } diff --git a/build/fbcode_builder/specs/wangle.py b/build/fbcode_builder/specs/wangle.py new file mode 100644 index 000000000000..62b5b3c867ce --- /dev/null +++ b/build/fbcode_builder/specs/wangle.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fizz as fizz +import specs.fmt as fmt +import specs.folly as folly +import specs.gmock as gmock +import specs.sodium as sodium + + +def fbcode_builder_spec(builder): + # Projects that **depend** on wangle need not spend time on tests. + builder.add_option( + "wangle/wangle/build:cmake_defines", + { + # This is set to ON in the wangle `fbcode_builder_config.py` + "BUILD_TESTS": "OFF" + }, + ) + return { + "depends_on": [gmock, fmt, folly, fizz, sodium], + "steps": [builder.fb_github_cmake_install("wangle/wangle/build")], + } diff --git a/build/fbcode_builder/specs/zstd.py b/build/fbcode_builder/specs/zstd.py new file mode 100644 index 000000000000..14d9a1249d0c --- /dev/null +++ b/build/fbcode_builder/specs/zstd.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + # This API should change rarely, so build the latest tag instead of master. + builder.add_option( + "facebook/zstd:git_hash", + ShellQuoted("$(git describe --abbrev=0 --tags origin/master)"), + ) + return { + "steps": [ + builder.github_project_workdir("facebook/zstd", "."), + builder.step( + "Build and install zstd", + [ + builder.make_and_install( + make_vars={ + "PREFIX": builder.option("prefix"), + } + ) + ], + ), + ], + } diff --git a/build/fbcode_builder/travis_docker_build.sh b/build/fbcode_builder/travis_docker_build.sh new file mode 100755 index 000000000000..d4cba10ef8f6 --- /dev/null +++ b/build/fbcode_builder/travis_docker_build.sh @@ -0,0 +1,42 @@ +#!/bin/bash -uex +# Copyright (c) Facebook, Inc. and its affiliates. +# .travis.yml in the top-level dir explains why this is a separate script. +# Read the docs: ./make_docker_context.py --help + +os_image=${os_image?Must be set by Travis} +gcc_version=${gcc_version?Must be set by Travis} +make_parallelism=${make_parallelism:-4} +# ccache is off unless requested +travis_cache_dir=${travis_cache_dir:-} +# The docker build never times out, unless specified +docker_build_timeout=${docker_build_timeout:-} + +cur_dir="$(realpath "$(dirname "$0")")" + +if [[ "$travis_cache_dir" == "" ]]; then + echo "ccache disabled, enable by setting env. var. travis_cache_dir" + ccache_tgz="" +elif [[ -e "$travis_cache_dir/ccache.tgz" ]]; then + ccache_tgz="$travis_cache_dir/ccache.tgz" +else + echo "$travis_cache_dir/ccache.tgz does not exist, starting with empty cache" + ccache_tgz=$(mktemp) + tar -T /dev/null -czf "$ccache_tgz" +fi + +docker_context_dir=$( + cd "$cur_dir/.." # Let the script find our fbcode_builder_config.py + "$cur_dir/make_docker_context.py" \ + --os-image "$os_image" \ + --gcc-version "$gcc_version" \ + --make-parallelism "$make_parallelism" \ + --local-repo-dir "$cur_dir/../.." \ + --ccache-tgz "$ccache_tgz" +) +cd "${docker_context_dir?Failed to make Docker context directory}" + +# Make it safe to iterate on the .sh in the tree while the script runs. +cp "$cur_dir/docker_build_with_ccache.sh" . +exec ./docker_build_with_ccache.sh \ + --build-timeout "$docker_build_timeout" \ + "$travis_cache_dir" diff --git a/build/fbcode_builder/utils.py b/build/fbcode_builder/utils.py new file mode 100644 index 000000000000..02459a200d4b --- /dev/null +++ b/build/fbcode_builder/utils.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +"Miscellaneous utility functions." + +import itertools +import logging +import os +import shutil +import subprocess +import sys +from contextlib import contextmanager + + +def recursively_flatten_list(l): + return itertools.chain.from_iterable( + (recursively_flatten_list(i) if type(i) is list else (i,)) for i in l + ) + + +def run_command(*cmd, **kwargs): + "The stdout of most fbcode_builder utilities is meant to be parsed." + logging.debug("Running: {0} with {1}".format(cmd, kwargs)) + kwargs["stdout"] = sys.stderr + subprocess.check_call(cmd, **kwargs) + + +@contextmanager +def make_temp_dir(d): + os.mkdir(d) + try: + yield d + finally: + shutil.rmtree(d, ignore_errors=True) + + +def _inner_read_config(path): + """ + Helper to read a named config file. + The grossness with the global is a workaround for this python bug: + https://bugs.python.org/issue21591 + The bug prevents us from defining either a local function or a lambda + in the scope of read_fbcode_builder_config below. + """ + global _project_dir + full_path = os.path.join(_project_dir, path) + return read_fbcode_builder_config(full_path) + + +def read_fbcode_builder_config(filename): + # Allow one spec to read another + # When doing so, treat paths as relative to the config's project directory. + # _project_dir is a "local" for _inner_read_config; see the comments + # in that function for an explanation of the use of global. + global _project_dir + _project_dir = os.path.dirname(filename) + + scope = {"read_fbcode_builder_config": _inner_read_config} + with open(filename) as config_file: + code = compile(config_file.read(), filename, mode="exec") + exec(code, scope) + return scope["config"] + + +def steps_for_spec(builder, spec, processed_modules=None): + """ + Sets `builder` configuration, and returns all the builder steps + necessary to build `spec` and its dependencies. + + Traverses the dependencies in depth-first order, honoring the sequencing + in each 'depends_on' list. + """ + if processed_modules is None: + processed_modules = set() + steps = [] + for module in spec.get("depends_on", []): + if module not in processed_modules: + processed_modules.add(module) + steps.extend( + steps_for_spec( + builder, module.fbcode_builder_spec(builder), processed_modules + ) + ) + steps.extend(spec.get("steps", [])) + return steps + + +def build_fbcode_builder_config(config): + return lambda builder: builder.build( + steps_for_spec(builder, config["fbcode_builder_spec"](builder)) + ) diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/license.header b/license.header new file mode 100644 index 000000000000..a2188c0de11b --- /dev/null +++ b/license.header @@ -0,0 +1,11 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt new file mode 100644 index 000000000000..6a4085713e39 --- /dev/null +++ b/scripts/CMakeLists.txt @@ -0,0 +1,12 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +add_subdirectory(tests) diff --git a/scripts/cci.py b/scripts/cci.py new file mode 100755 index 000000000000..b99e763a836e --- /dev/null +++ b/scripts/cci.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys +import json + +import util + +CIRCLECI_V2 = "https://circleci.com/api/v2" +CIRCLECI_V1 = "https://circleci.com/api/v1.1" + + +def get_circleci_token(): + token = os.getenv("CIRCLECI_API_TOKEN") + + if token is None: + print( + """ + CIRCLECI_API_TOKEN is not set in the environment. An API key is + required to access CircleCi. You can create an API token here : + + https://circleci.com/account/api + + Then set this value in your .profile or .bashrc. + + export CIRCLECI_API_TOKEN="Your Token Value Here" +""", + file=sys.stderr, + ) + + sys.exit(1) + + return token + + +def circleci_url(token, url, compressed=False): + return util.run( + f"curl -s --header 'Circle-Token: {token}' \ + --header 'Accept: application/json' \ + --header 'Content-Type: application/json' \ + '{url}'", + compressed=compressed, + )[1] + + +def get_pr_workflows(token, repo, pr): + url = f"{CIRCLECI_V2}/insights/gh/{repo}/workflows/dist-compile?branch=pull/{pr}" + + return circleci_url(token, url).json() + + +def get_workflow_jobs(token, id): + url = f"{CIRCLECI_V2}/workflow/{id}/job" + + return circleci_url(token, url).json() + + +def get_job_steps(token, repo, job): + url = f"{CIRCLECI_V1}/project/gh/{repo}/{job}" + + return circleci_url(token, url).json() + + +def get_action_output(token, url): + return circleci_url(token, url, compressed=True).json() + + +def output(args): + + github_upstream = util.run("git remote get-url --push upstream")[1].extract( + r".*:(.*)[.]git" + ) + github_user = util.run("git remote get-url --push origin")[1].extract(r".*:(.*)/") + git_branch = util.run("git symbolic-ref --short HEAD")[1] + + if args.pr: + pull_request = args.pr + else: + stdout = util.run(f"hub pr list -h {github_user}:{git_branch}")[1] + if stdout == "": + print(f"Error: no pull request for {github_user}:{git_branch}") + sys.exit(1) + + pull_request = stdout.extract(r".*#([0-9]+) .*") + + if args.token is None: + token = get_circleci_token() + else: + token = args.token + + pr_workflows = get_pr_workflows(token, github_upstream, pull_request) + latest_workflow = sorted( + pr_workflows["items"], key=lambda x: x.created_at, reverse=True + )[0] + latest_workflow_id = latest_workflow.id + + workflow_jobs = get_workflow_jobs(token, latest_workflow_id) + + for job in workflow_jobs["items"]: + if job.status != args.job_status or args.job_name not in job.name: + continue + + steps = get_job_steps(token, github_upstream, job.job_number).steps + for step in steps: + if args.step_name not in step.name.lower(): + continue + + for action in step.actions: + if (args.step_status == "failed" and action.failed) or ( + args.step_status != "failed" and action.failed is None + ): + for item in get_action_output(token, action.output_url): + print(item.message) + + return 0 + + +def help(args): + parser.print_help() + + return 0 + + +def parse_args(): + global parser + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="""CircleCi utility + + Output subcommand: + + To obtain the output from failiing build jobs for the PR associated with + the current checkout: + + > cci.py output + + Use the ----job-name and --step-name option to obtain output from failing + 'check' jobs: + + > cci.py output --job-name check --step-name check + + Specify a particular job name for obtain output from a single job: + + > ./scripts/cci.py output --job-name format-check --step-name check + +""", + ) + parser.add_argument("--token", help="CircleCi API token") + + command = parser.add_subparsers(dest="command") + output_command_parser = command.add_parser("output") + output_command_parser.add_argument("--pr") + output_command_parser.add_argument("--job-name", default="build") + output_command_parser.add_argument("--job-status", default="failed") + output_command_parser.add_argument("--step-name", default="build") + output_command_parser.add_argument("--step-status", default="failed") + + parser.set_defaults(command="help") + + return parser.parse_args() + + +def main(): + args = parse_args() + return globals()[args.command](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-container.dockfile b/scripts/check-container.dockfile new file mode 100644 index 000000000000..5f402f0e71eb --- /dev/null +++ b/scripts/check-container.dockfile @@ -0,0 +1,14 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM ubuntu:hirsute +COPY setup-check.sh /root +RUN bash /root/setup-check.sh diff --git a/scripts/check.py b/scripts/check.py new file mode 100755 index 000000000000..c1ad07d97ce4 --- /dev/null +++ b/scripts/check.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +from collections import OrderedDict +import os +import regex +import subprocess +import sys + +from util import attrdict +import util + +EXTENSIONS = "cpp,h,inc,prolog" +SCRIPTS = util.script_path() + + +def get_diff(file, formatted): + if not formatted.endswith("\n"): + formatted = formatted + "\n" + + status, stdout, stderr = util.run( + f"diff -u {file} --label {file} --label {file} -", input=formatted + ) + if stdout != "": + stdout = f"diff a/{file} b/{file}\n" + stdout + + return status, stdout, stderr + + +class CppFormatter(str): + def diff(self, commit): + if commit == "": + return get_diff(self, util.run(f"clang-format --style=file {self}")[1]) + else: + return util.run( + f"{SCRIPTS}/git-clang-format -q --extensions='{EXTENSIONS}' --diff --style=file {commit} {self}" + ) + + def fix(self, commit): + if commit == "": + return util.run(f"clang-format -i --style=file {self}")[0] == 0 + else: + return ( + util.run( + f"{SCRIPTS}/git-clang-format -q --extensions='{EXTENSIONS}' --style=file {commit} {self}" + )[0] + == 0 + ) + + +class CMakeFormatter(str): + def diff(self, commit): + return get_diff( + self, util.run(f"cmake-format --first-comment-is-literal True {self}")[1] + ) + + def fix(self, commit): + return ( + util.run(f"cmake-format --first-comment-is-literal True -i {self}")[0] == 0 + ) + + +class PythonFormatter(str): + def diff(self, commit): + return util.run(f"black -q --diff {self}") + + def fix(self, commit): + return util.run(f"black -q {self}")[0] == 0 + + +format_file_types = OrderedDict( + { + "CMakeLists.txt": attrdict({"formatter": CMakeFormatter}), + "*.cpp": attrdict({"formatter": CppFormatter}), + "*.h": attrdict({"formatter": CppFormatter}), + "*.inc": attrdict({"formatter": CppFormatter}), + "*.prolog": attrdict({"formatter": CppFormatter}), + "*.py": attrdict({"formatter": PythonFormatter}), + } +) + + +def get_formatter(filename): + if filename in format_file_types: + return format_file_types[filename] + + return format_file_types.get("*" + util.get_fileextn(filename), None) + + +def format_command(commit, files, fix): + ok = 0 + for filepath in files: + filename = util.get_filename(filepath) + filetype = get_formatter(filename) + + if filetype is None: + print("Skip : " + filepath, file=sys.stderr) + continue + + file = filetype.formatter(filepath) + + if fix == "show": + status, diff, stderr = file.diff(commit) + + if stderr != "": + ok = 1 + print(f"Error: {file}", file=sys.stderr) + continue + + if diff != "" and diff != "no modified files to format": + ok = 1 + print(f"Fix : {file}", file=sys.stderr) + print(diff) + else: + print(f"Ok : {file}", file=sys.stderr) + + else: + print(f"Fix : {file}", file=sys.stderr) + if not file.fix(commit): + ok = 1 + print(f"Error: {file}", file=sys.stderr) + + return ok + + +def header_command(commit, files, fix): + options = "-vk" if fix == "show" else "-i" + + status, stdout, stderr = util.run( + f"{SCRIPTS}/license-header.py {options} -", input=files + ) + + if stdout != "": + print(stdout) + + return status + + +def tidy_command(commit, files, fix): + files = [file for file in files if regex.match(r".*\.cpp$", file)] + + if not files: + return 0 + + commit = f"--commit {commit}" if commit != "" else "" + fix = "--fix" if fix == "fix" else "" + + status, stdout, stderr = util.run( + f"{SCRIPTS}/run-clang-tidy.py {commit} {fix} -", input=files + ) + + if stdout != "": + print(stdout) + + return status + + +def get_commit(files): + if files == "commit": + return "HEAD^" + + if files == "branch": + return util.run("git merge-base origin/master HEAD")[1] + + return "" + + +def get_files(commit, path): + filelist = [] + + if commit != "": + status, stdout, stderr = util.run( + f"git diff --name-only --diff-filter='ACM' {commit}" + ) + filelist = stdout.splitlines() + else: + for root, dirs, files in os.walk(path): + for name in files: + filelist.append(os.path.join(root, name)) + + return [ + file + for file in filelist + if "/data/" not in file + and "velox/external/" not in file + and "build/fbcode_builder" not in file + and "build/deps" not in file + and "cmake-build-debug" not in file + ] + + +def help(args): + parser.print_help() + return 0 + + +def add_check_options(subparser, name): + parser = subparser.add_parser(name) + parser.add_argument("--fix", action="store_const", default="show", const="fix") + return parser + + +def add_options(parser): + files = parser.add_subparsers(dest="files") + + tree_parser = add_check_options(files, "tree") + tree_parser.add_argument("path", default="") + + branch_parser = add_check_options(files, "branch") + commit_parser = add_check_options(files, "commit") + + +def add_check_command(parser, name): + subparser = parser.add_parser(name) + add_options(subparser) + + return subparser + + +def parse_args(): + global parser + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="""Check format/header/tidy + + check.py {format,header,tidy} {commit,branch} [--fix] + check.py {format,header,tidy} {tree} [--fix] PATH +""", + ) + command = parser.add_subparsers(dest="command") + command.add_parser("help") + + format_command_parser = add_check_command(command, "format") + header_command_parser = add_check_command(command, "header") + tidy_command_parser = add_check_command(command, "tidy") + + parser.set_defaults(path="") + parser.set_defaults(command="help") + + return parser.parse_args() + + +def run_command(args, command): + commit = get_commit(args.files) + files = get_files(commit, args.path) + + return command(commit, files, args.fix) + + +def format(args): + return run_command(args, format_command) + + +def header(args): + return run_command(args, header_command) + + +def tidy(args): + return run_command(args, tidy_command) + + +def main(): + args = parse_args() + return globals()[args.command](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/circleci-container.dockfile b/scripts/circleci-container.dockfile new file mode 100644 index 000000000000..d7133a80c088 --- /dev/null +++ b/scripts/circleci-container.dockfile @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Build the test and build container for presto_cpp +# +FROM centos:centos8 +COPY setup-circleci.sh / +RUN mkdir build && ( cd build && bash /setup-circleci.sh ) && rm -rf build diff --git a/scripts/git-clang-format b/scripts/git-clang-format new file mode 100755 index 000000000000..46e7f5cd0ca7 --- /dev/null +++ b/scripts/git-clang-format @@ -0,0 +1,622 @@ +#!/usr/bin/env python3 +# +#===- git-clang-format - ClangFormat Git Integration ---------*- python -*--===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#============================================================================== +#LLVM Release License +#============================================================================== +#University of Illinois/NCSA +#Open Source License +# +#Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign. +#All rights reserved. +# +#Developed by: +# +# LLVM Team +# +# University of Illinois at Urbana-Champaign +# +# http://llvm.org +# +#Permission is hereby granted, free of charge, to any person obtaining a copy of +#this software and associated documentation files (the "Software"), to deal with +#the Software without restriction, including without limitation the rights to +#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +#of the Software, and to permit persons to whom the Software is furnished to do +#so, subject to the following conditions: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimers. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimers in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the names of the LLVM Team, University of Illinois at +# Urbana-Champaign, nor the names of its contributors may be used to +# endorse or promote products derived from this Software without specific +# prior written permission. +# +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +#CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +#SOFTWARE. +#===------------------------------------------------------------------------===# + +r""" +clang-format git integration +============================ + +This file provides a clang-format integration for git. Put it somewhere in your +path and ensure that it is executable. Then, "git clang-format" will invoke +clang-format on the changes in current files or a specific commit. + +For further details, run: +git clang-format -h + +Requires Python 2.7 or Python 3 +""" + +from __future__ import absolute_import, division, print_function +import argparse +import collections +import contextlib +import errno +import os +import re +import subprocess +import sys + +usage = 'git clang-format [OPTIONS] [] [] [--] [...]' + +desc = ''' +If zero or one commits are given, run clang-format on all lines that differ +between the working directory and , which defaults to HEAD. Changes are +only applied to the working directory. + +If two commits are given (requires --diff), run clang-format on all lines in the +second that differ from the first . + +The following git-config settings set the default of the corresponding option: + clangFormat.binary + clangFormat.commit + clangFormat.extension + clangFormat.style +''' + +# Name of the temporary index file in which save the output of clang-format. +# This file is created within the .git directory. +temp_index_basename = 'clang-format-index' + + +Range = collections.namedtuple('Range', 'start, count') + + +def main(): + config = load_git_config() + + # In order to keep '--' yet allow options after positionals, we need to + # check for '--' ourselves. (Setting nargs='*' throws away the '--', while + # nargs=argparse.REMAINDER disallows options after positionals.) + argv = sys.argv[1:] + try: + idx = argv.index('--') + except ValueError: + dash_dash = [] + else: + dash_dash = argv[idx:] + argv = argv[:idx] + + default_extensions = ','.join([ + # From clang/lib/Frontend/FrontendOptions.cpp, all lower case + 'c', 'h', # C + 'm', # ObjC + 'mm', # ObjC++ + 'cc', 'cp', 'cpp', 'c++', 'cxx', 'hpp', # C++ + 'cu', # CUDA + # Other languages that clang-format supports + 'proto', 'protodevel', # Protocol Buffers + 'java', # Java + 'js', # JavaScript + 'ts', # TypeScript + ]) + + p = argparse.ArgumentParser( + usage=usage, formatter_class=argparse.RawDescriptionHelpFormatter, + description=desc) + p.add_argument('--binary', + default=config.get('clangformat.binary', 'clang-format'), + help='path to clang-format'), + p.add_argument('--commit', + default=config.get('clangformat.commit', 'HEAD'), + help='default commit to use if none is specified'), + p.add_argument('--diff', action='store_true', + help='print a diff instead of applying the changes') + p.add_argument('--extensions', + default=config.get('clangformat.extensions', + default_extensions), + help=('comma-separated list of file extensions to format, ' + 'excluding the period and case-insensitive')), + p.add_argument('-f', '--force', action='store_true', + help='allow changes to unstaged files') + p.add_argument('-p', '--patch', action='store_true', + help='select hunks interactively') + p.add_argument('-q', '--quiet', action='count', default=0, + help='print less information') + p.add_argument('--style', + default=config.get('clangformat.style', None), + help='passed to clang-format'), + p.add_argument('-v', '--verbose', action='count', default=0, + help='print extra information') + # We gather all the remaining positional arguments into 'args' since we need + # to use some heuristics to determine whether or not was present. + # However, to print pretty messages, we make use of metavar and help. + p.add_argument('args', nargs='*', metavar='', + help='revision from which to compute the diff') + p.add_argument('ignored', nargs='*', metavar='...', + help='if specified, only consider differences in these files') + opts = p.parse_args(argv) + + opts.verbose -= opts.quiet + del opts.quiet + + commits, files = interpret_args(opts.args, dash_dash, opts.commit) + if len(commits) > 1: + if not opts.diff: + die('--diff is required when two commits are given') + else: + if len(commits) > 2: + die('at most two commits allowed; %d given' % len(commits)) + changed_lines = compute_diff_and_extract_lines(commits, files) + if opts.verbose >= 1: + ignored_files = set(changed_lines) + filter_by_extension(changed_lines, opts.extensions.lower().split(',')) + if opts.verbose >= 1: + ignored_files.difference_update(changed_lines) + if ignored_files: + print('Ignoring changes in the following files (wrong extension):') + for filename in ignored_files: + print(' %s' % filename) + if changed_lines: + print('Running clang-format on the following files:') + for filename in changed_lines: + print(' %s' % filename) + if not changed_lines: + print('no modified files to format') + return + # The computed diff outputs absolute paths, so we must cd before accessing + # those files. + cd_to_toplevel() + if len(commits) > 1: + old_tree = commits[1] + new_tree = run_clang_format_and_save_to_tree(changed_lines, + revision=commits[1], + binary=opts.binary, + style=opts.style) + else: + old_tree = create_tree_from_workdir(changed_lines) + new_tree = run_clang_format_and_save_to_tree(changed_lines, + binary=opts.binary, + style=opts.style) + if opts.verbose >= 1: + print('old tree: %s' % old_tree) + print('new tree: %s' % new_tree) + if old_tree == new_tree: + if opts.verbose >= 0: + print('clang-format did not modify any files') + elif opts.diff: + print_diff(old_tree, new_tree) + else: + changed_files = apply_changes(old_tree, new_tree, force=opts.force, + patch_mode=opts.patch) + if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1: + print('changed files:') + for filename in changed_files: + print(' %s' % filename) + + +def load_git_config(non_string_options=None): + """Return the git configuration as a dictionary. + + All options are assumed to be strings unless in `non_string_options`, in which + is a dictionary mapping option name (in lower case) to either "--bool" or + "--int".""" + if non_string_options is None: + non_string_options = {} + out = {} + for entry in run('git', 'config', '--list', '--null').split('\0'): + if entry: + name, value = entry.split('\n', 1) + if name in non_string_options: + value = run('git', 'config', non_string_options[name], name) + out[name] = value + return out + + +def interpret_args(args, dash_dash, default_commit): + """Interpret `args` as "[commits] [--] [files]" and return (commits, files). + + It is assumed that "--" and everything that follows has been removed from + args and placed in `dash_dash`. + + If "--" is present (i.e., `dash_dash` is non-empty), the arguments to its + left (if present) are taken as commits. Otherwise, the arguments are checked + from left to right if they are commits or files. If commits are not given, + a list with `default_commit` is used.""" + if dash_dash: + if len(args) == 0: + commits = [default_commit] + else: + commits = args + for commit in commits: + object_type = get_object_type(commit) + if object_type not in ('commit', 'tag'): + if object_type is None: + die("'%s' is not a commit" % commit) + else: + die("'%s' is a %s, but a commit was expected" % (commit, object_type)) + files = dash_dash[1:] + elif args: + commits = [] + while args: + if not disambiguate_revision(args[0]): + break + commits.append(args.pop(0)) + if not commits: + commits = [default_commit] + files = args + else: + commits = [default_commit] + files = [] + return commits, files + + +def disambiguate_revision(value): + """Returns True if `value` is a revision, False if it is a file, or dies.""" + # If `value` is ambiguous (neither a commit nor a file), the following + # command will die with an appropriate error message. + run('git', 'rev-parse', value, verbose=False) + object_type = get_object_type(value) + if object_type is None: + return False + if object_type in ('commit', 'tag'): + return True + die('`%s` is a %s, but a commit or filename was expected' % + (value, object_type)) + + +def get_object_type(value): + """Returns a string description of an object's type, or None if it is not + a valid git object.""" + cmd = ['git', 'cat-file', '-t', value] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.returncode != 0: + return None + return convert_string(stdout.strip()) + + +def compute_diff_and_extract_lines(commits, files): + """Calls compute_diff() followed by extract_lines().""" + diff_process = compute_diff(commits, files) + changed_lines = extract_lines(diff_process.stdout) + diff_process.stdout.close() + diff_process.wait() + if diff_process.returncode != 0: + # Assume error was already printed to stderr. + sys.exit(2) + return changed_lines + + +def compute_diff(commits, files): + """Return a subprocess object producing the diff from `commits`. + + The return value's `stdin` file object will produce a patch with the + differences between the working directory and the first commit if a single + one was specified, or the difference between both specified commits, filtered + on `files` (if non-empty). Zero context lines are used in the patch.""" + git_tool = 'diff-index' + if len(commits) > 1: + git_tool = 'diff-tree' + cmd = ['git', git_tool, '-p', '-U0'] + commits + ['--'] + cmd.extend(files) + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.stdin.close() + return p + + +def extract_lines(patch_file): + """Extract the changed lines in `patch_file`. + + The return value is a dictionary mapping filename to a list of (start_line, + line_count) pairs. + + The input must have been produced with ``-U0``, meaning unidiff format with + zero lines of context. The return value is a dict mapping filename to a + list of line `Range`s.""" + matches = {} + for line in patch_file: + line = convert_string(line) + match = re.search(r'^\+\+\+\ [^/]+/(.*)', line) + if match: + filename = match.group(1).rstrip('\r\n') + match = re.search(r'^@@ -[0-9,]+ \+(\d+)(,(\d+))?', line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + if line_count > 0: + matches.setdefault(filename, []).append(Range(start_line, line_count)) + return matches + + +def filter_by_extension(dictionary, allowed_extensions): + """Delete every key in `dictionary` that doesn't have an allowed extension. + + `allowed_extensions` must be a collection of lowercase file extensions, + excluding the period.""" + allowed_extensions = frozenset(allowed_extensions) + for filename in list(dictionary.keys()): + base_ext = filename.rsplit('.', 1) + if len(base_ext) == 1 and '' in allowed_extensions: + continue + if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions: + del dictionary[filename] + + +def cd_to_toplevel(): + """Change to the top level of the git repository.""" + toplevel = run('git', 'rev-parse', '--show-toplevel') + os.chdir(toplevel) + + +def create_tree_from_workdir(filenames): + """Create a new git tree with the given files from the working directory. + + Returns the object ID (SHA-1) of the created tree.""" + return create_tree(filenames, '--stdin') + + +def run_clang_format_and_save_to_tree(changed_lines, revision=None, + binary='clang-format', style=None): + """Run clang-format on each file and save the result to a git tree. + + Returns the object ID (SHA-1) of the created tree.""" + def iteritems(container): + try: + return container.iteritems() # Python 2 + except AttributeError: + return container.items() # Python 3 + def index_info_generator(): + for filename, line_ranges in iteritems(changed_lines): + if revision: + git_metadata_cmd = ['git', 'ls-tree', + '%s:%s' % (revision, os.path.dirname(filename)), + os.path.basename(filename)] + git_metadata = subprocess.Popen(git_metadata_cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout = git_metadata.communicate()[0] + mode = oct(int(stdout.split()[0], 8)) + else: + mode = oct(os.stat(filename).st_mode) + # Adjust python3 octal format so that it matches what git expects + if mode.startswith('0o'): + mode = '0' + mode[2:] + blob_id = clang_format_to_blob(filename, line_ranges, + revision=revision, + binary=binary, + style=style) + yield '%s %s\t%s' % (mode, blob_id, filename) + return create_tree(index_info_generator(), '--index-info') + + +def create_tree(input_lines, mode): + """Create a tree object from the given input. + + If mode is '--stdin', it must be a list of filenames. If mode is + '--index-info' is must be a list of values suitable for "git update-index + --index-info", such as " ". Any other mode + is invalid.""" + assert mode in ('--stdin', '--index-info') + cmd = ['git', 'update-index', '--add', '-z', mode] + with temporary_index_file(): + p = subprocess.Popen(cmd, stdin=subprocess.PIPE) + for line in input_lines: + p.stdin.write(to_bytes('%s\0' % line)) + p.stdin.close() + if p.wait() != 0: + die('`%s` failed' % ' '.join(cmd)) + tree_id = run('git', 'write-tree') + return tree_id + + +def clang_format_to_blob(filename, line_ranges, revision=None, + binary='clang-format', style=None): + """Run clang-format on the given file and save the result to a git blob. + + Runs on the file in `revision` if not None, or on the file in the working + directory if `revision` is None. + + Returns the object ID (SHA-1) of the created blob.""" + clang_format_cmd = [binary] + if style: + clang_format_cmd.extend(['-style='+style]) + clang_format_cmd.extend([ + '-lines=%s:%s' % (start_line, start_line+line_count-1) + for start_line, line_count in line_ranges]) + if revision: + clang_format_cmd.extend(['-assume-filename='+filename]) + git_show_cmd = ['git', 'cat-file', 'blob', '%s:%s' % (revision, filename)] + git_show = subprocess.Popen(git_show_cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + git_show.stdin.close() + clang_format_stdin = git_show.stdout + else: + clang_format_cmd.extend([filename]) + git_show = None + clang_format_stdin = subprocess.PIPE + try: + clang_format = subprocess.Popen(clang_format_cmd, stdin=clang_format_stdin, + stdout=subprocess.PIPE) + if clang_format_stdin == subprocess.PIPE: + clang_format_stdin = clang_format.stdin + except OSError as e: + if e.errno == errno.ENOENT: + die('cannot find executable "%s"' % binary) + else: + raise + clang_format_stdin.close() + hash_object_cmd = ['git', 'hash-object', '-w', '--path='+filename, '--stdin'] + hash_object = subprocess.Popen(hash_object_cmd, stdin=clang_format.stdout, + stdout=subprocess.PIPE) + clang_format.stdout.close() + stdout = hash_object.communicate()[0] + if hash_object.returncode != 0: + die('`%s` failed' % ' '.join(hash_object_cmd)) + if clang_format.wait() != 0: + die('`%s` failed' % ' '.join(clang_format_cmd)) + if git_show and git_show.wait() != 0: + die('`%s` failed' % ' '.join(git_show_cmd)) + return convert_string(stdout).rstrip('\r\n') + + +@contextlib.contextmanager +def temporary_index_file(tree=None): + """Context manager for setting GIT_INDEX_FILE to a temporary file and deleting + the file afterward.""" + index_path = create_temporary_index(tree) + old_index_path = os.environ.get('GIT_INDEX_FILE') + os.environ['GIT_INDEX_FILE'] = index_path + try: + yield + finally: + if old_index_path is None: + del os.environ['GIT_INDEX_FILE'] + else: + os.environ['GIT_INDEX_FILE'] = old_index_path + os.remove(index_path) + + +def create_temporary_index(tree=None): + """Create a temporary index file and return the created file's path. + + If `tree` is not None, use that as the tree to read in. Otherwise, an + empty index is created.""" + gitdir = run('git', 'rev-parse', '--git-dir') + path = os.path.join(gitdir, temp_index_basename) + if tree is None: + tree = '--empty' + run('git', 'read-tree', '--index-output='+path, tree) + return path + + +def print_diff(old_tree, new_tree): + """Print the diff between the two trees to stdout.""" + # We use the porcelain 'diff' and not plumbing 'diff-tree' because the output + # is expected to be viewed by the user, and only the former does nice things + # like color and pagination. + # + # We also only print modified files since `new_tree` only contains the files + # that were modified, so unmodified files would show as deleted without the + # filter. + subprocess.check_call(['git', 'diff', '--diff-filter=M', old_tree, new_tree, + '--']) + + +def apply_changes(old_tree, new_tree, force=False, patch_mode=False): + """Apply the changes in `new_tree` to the working directory. + + Bails if there are local changes in those files and not `force`. If + `patch_mode`, runs `git checkout --patch` to select hunks interactively.""" + changed_files = run('git', 'diff-tree', '--diff-filter=M', '-r', '-z', + '--name-only', old_tree, + new_tree).rstrip('\0').split('\0') + if not force: + unstaged_files = run('git', 'diff-files', '--name-status', *changed_files) + if unstaged_files: + print('The following files would be modified but ' + 'have unstaged changes:', file=sys.stderr) + print(unstaged_files, file=sys.stderr) + print('Please commit, stage, or stash them first.', file=sys.stderr) + sys.exit(2) + if patch_mode: + # In patch mode, we could just as well create an index from the new tree + # and checkout from that, but then the user will be presented with a + # message saying "Discard ... from worktree". Instead, we use the old + # tree as the index and checkout from new_tree, which gives the slightly + # better message, "Apply ... to index and worktree". This is not quite + # right, since it won't be applied to the user's index, but oh well. + with temporary_index_file(old_tree): + subprocess.check_call(['git', 'checkout', '--patch', new_tree]) + index_tree = old_tree + else: + with temporary_index_file(new_tree): + run('git', 'checkout-index', '-a', '-f') + return changed_files + + +def run(*args, **kwargs): + stdin = kwargs.pop('stdin', '') + verbose = kwargs.pop('verbose', True) + strip = kwargs.pop('strip', True) + for name in kwargs: + raise TypeError("run() got an unexpected keyword argument '%s'" % name) + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + stdout, stderr = p.communicate(input=stdin) + + stdout = convert_string(stdout) + stderr = convert_string(stderr) + + if p.returncode == 0: + if stderr: + if verbose: + print('`%s` printed to stderr:' % ' '.join(args), file=sys.stderr) + print(stderr.rstrip(), file=sys.stderr) + if strip: + stdout = stdout.rstrip('\r\n') + return stdout + if verbose: + print('`%s` returned %s' % (' '.join(args), p.returncode), file=sys.stderr) + if stderr: + print(stderr.rstrip(), file=sys.stderr) + sys.exit(2) + + +def die(message): + print('error:', message, file=sys.stderr) + sys.exit(2) + + +def to_bytes(str_input): + # Encode to UTF-8 to get binary data. + if isinstance(str_input, bytes): + return str_input + return str_input.encode('utf-8') + + +def to_string(bytes_input): + if isinstance(bytes_input, str): + return bytes_input + return bytes_input.encode('utf-8') + + +def convert_string(bytes_input): + try: + return to_string(bytes_input.decode('utf-8')) + except AttributeError: # 'str' object has no attribute 'decode'. + return str(bytes_input) + except UnicodeError: + return str(bytes_input) + +if __name__ == '__main__': + main() diff --git a/scripts/license-header.py b/scripts/license-header.py new file mode 100755 index 000000000000..c91e3a9256a7 --- /dev/null +++ b/scripts/license-header.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from collections import OrderedDict +import fnmatch +import os +import regex +import sys + + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + + +def parse_args(): + parser = argparse.ArgumentParser(description="Update license headers") + parser.add_argument("--header", default="license.header", help="header file") + parser.add_argument( + "--extra", + default=80, + help="extra characters past beginning of file to look for header", + ) + parser.add_argument( + "--editdist", default=12, type=int, help="max edit distance between headers" + ) + parser.add_argument( + "--remove", default=False, action="store_true", help="remove the header" + ) + parser.add_argument( + "--cslash", + default=False, + action="store_true", + help='use C slash "//" style comments', + ) + parser.add_argument( + "-v", default=False, action="store_true", dest="verbose", help="verbose output" + ) + + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-k", default=False, action="store_true", dest="check", help="check headers" + ) + group.add_argument( + "-i", + default=False, + action="store_true", + dest="inplace", + help="edit file inplace", + ) + + parser.add_argument("files", metavar="FILES", nargs="+", help="files to process") + + return parser.parse_args() + + +def file_read(filename): + with open(filename) as file: + return file.read() + + +def file_lines(filename): + return file_read(filename).rstrip().split("\n") + + +def wrapper(prefix, leader, suffix, header): + return prefix + "\n".join([leader + line for line in header]) + suffix + + +def wrapper_chpp(header, args): + if args.cslash: + return wrapper("", "//", "\n", header) + else: + return wrapper("/*\n", " *", "\n */\n", header) + + +def wrapper_hash(header, args): + return wrapper("", "#", "\n", header) + + +file_types = OrderedDict( + { + "CMakeLists.txt": attrdict({"wrapper": wrapper_hash, "hashbang": False}), + "Makefile": attrdict({"wrapper": wrapper_hash, "hashbang": False}), + "*.cpp": attrdict({"wrapper": wrapper_chpp, "hashbang": False}), + "*.dockfile": attrdict({"wrapper": wrapper_hash, "hashbang": False}), + "*.h": attrdict({"wrapper": wrapper_chpp, "hashbang": False}), + "*.inc": attrdict({"wrapper": wrapper_chpp, "hashbang": False}), + "*.java": attrdict({"wrapper": wrapper_chpp, "hashbang": False}), + "*.prolog": attrdict({"wrapper": wrapper_chpp, "hashbang": False}), + "*.py": attrdict({"wrapper": wrapper_hash, "hashbang": True}), + "*.sh": attrdict({"wrapper": wrapper_hash, "hashbang": True}), + "*.thrift": attrdict({"wrapper": wrapper_chpp, "hashbang": False}), + "*.txt": attrdict({"wrapper": wrapper_hash, "hashbang": True}), + "*.yml": attrdict({"wrapper": wrapper_hash, "hashbang": False}), + } +) + +file_pattern = regex.compile( + "|".join(["^" + fnmatch.translate(type) + "$" for type in file_types.keys()]) +) + + +def get_filename(filename): + return os.path.basename(filename) + + +def get_fileextn(filename): + split = os.path.splitext(filename) + if len(split) <= 1: + return "" + + return split[-1] + + +def get_wrapper(filename): + if filename in file_types: + return file_types[filename] + + return file_types["*" + get_fileextn(filename)] + + +def message(file, string): + if file: + print(string, file=file) + + +def main(): + fail = False + log_to = None + + args = parse_args() + + if args.verbose: + log_to = sys.stderr + + if args.check: + log_to = None + + if args.verbose: + log_to = sys.stdout + + header_text = file_lines(args.header) + + if len(args.files) == 1 and args.files[0] == "-": + files = [file.strip() for file in sys.stdin.readlines()] + else: + files = args.files + + for filepath in files: + filename = get_filename(filepath) + + matched = file_pattern.match(filename) + + if not matched: + message(log_to, "Skip : " + filepath) + continue + + content = file_read(filepath) + wrap = get_wrapper(filename) + + header_comment = wrap.wrapper(header_text, args) + + start = 0 + end = 0 + + # Look for an exact substr match + # + found = content.find(header_comment, 0, len(header_comment) + args.extra) + if found >= 0: + if not args.remove: + message(log_to, "OK : " + filepath) + continue + + start = found + end = found + len(header_comment) + else: + # Look for a fuzzy match in the first 60 chars + # + found = regex.search( + "(?be)(%s){e<=%d}" % (regex.escape(header_comment[0:60]), 6), + content[0 : 80 + args.extra], + ) + if found: + fuzzy = regex.compile( + "(?be)(%s){e<=%d}" % (regex.escape(header_comment), args.editdist) + ) + + # If the first 80 chars match - try harder for the rest of the header + # + found = fuzzy.search( + content[0 : len(header_comment) + args.extra], found.start() + ) + if found: + start = found.start() + end = found.end() + + if args.remove: + if start == 0 and end == 0: + if not args.inplace: + print(content, end="") + + message(log_to, "OK : " + filepath) + continue + + # If removing the header text, zero it out there. + header_comment = "" + + message(log_to, "Fix : " + filepath) + + if args.check: + fail = True + continue + + # Remove any partially matching header + # + content = content[0:start] + content[end:] + + if wrap.hashbang: + search = regex.search("^#!.*\n", content) + if search: + content = ( + content[search.start() : search.end()] + + header_comment + + content[search.end() :] + ) + else: + content = header_comment + content + else: + content = header_comment + content + + if args.inplace: + with open(filepath, "w") as file: + print(content, file=file, end="") + else: + print(content, end="") + + if fail: + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run-clang-tidy.py b/scripts/run-clang-tidy.py new file mode 100755 index 000000000000..c96bf609bb5c --- /dev/null +++ b/scripts/run-clang-tidy.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from itertools import groupby +import json +import regex +import sys + +import util + +CODE_CHECKS = """* + -abseil-* + -android-* + -cert-err58-cpp + -clang-analyzer-osx-* + -cppcoreguidelines-avoid-c-arrays + -cppcoreguidelines-avoid-magic-numbers + -cppcoreguidelines-pro-bounds-array-to-pointer-decay + -cppcoreguidelines-pro-bounds-pointer-arithmetic + -cppcoreguidelines-pro-type-reinterpret-cast + -cppcoreguidelines-pro-type-vararg + -fuchsia-* + -google-* + -hicpp-avoid-c-arrays + -hicpp-deprecated-headers + -hicpp-no-array-decay + -hicpp-use-equals-default + -hicpp-vararg + -llvmlibc-* + -llvm-header-guard + -llvm-include-order + -mpi-* + -misc-non-private-member-variables-in-classes + -misc-no-recursion + -misc-unused-parameters + -modernize-avoid-c-arrays + -modernize-deprecated-headers + -modernize-use-nodiscard + -modernize-use-trailing-return-type + -objc-* + -openmp-* + -readability-avoid-const-params-in-decls + -readability-convert-member-functions-to-static + -readability-magic-numbers + -zircon-* +""" + +# Additional opt-outs because googletest macros trip too many things. +# +TEST_CHECKS = ( + CODE_CHECKS + + """ + -cert-err58-cpp + -cppcoreguidelines-avoid-goto + -cppcoreguidelines-avoid-non-const-global-variables + -cppcoreguidelines-owning-memory + -cppcoreguidelines-pro-type-vararg + -cppcoreguidelines-special-member-functions + -hicpp-avoid-goto + -hicpp-special-member-functions + -hicpp-vararg + -misc-no-recursion + -readability-implicit-bool-conversion +""" +) + + +def check_list(check_string): + return ",".join([check.strip() for check in check_string.strip().splitlines()]) + + +CODE_CHECKS = check_list(CODE_CHECKS) +TEST_CHECKS = check_list(TEST_CHECKS) + + +class Multimap(dict): + def __setitem__(self, key, value): + if key not in self: + dict.__setitem__(self, key, [value]) # call super method to avoid recursion + else: + self[key].append(value) + + +def git_changed_lines(commit): + file = "" + changed_lines = Multimap() + + for line in util.run(f"git diff --text --unified=0 {commit}")[1].splitlines(): + line = line.rstrip("\n") + fields = line.split() + + match = regex.match(r"^\+\+\+ b/.*", line) + if match: + file = "" + + match = regex.match(r"^\+\+\+ b/(.*(\.cpp|\.h))$", line) + if match: + file = match.group(1) + + match = regex.match(r"^@@", line) + if match and file != "": + lspan = fields[2].split(",") + if len(lspan) <= 1: + lspan.append(0) + + changed_lines[file] = [int(lspan[0]), int(lspan[0]) + int(lspan[1])] + + return json.dumps( + [{"name": key, "lines": value} for key, value in changed_lines.items()] + ) + + +def checks(args): + status, stdout, stderr = util.run( + f"clang-tidy -checks='{CODE_CHECKS}' --list-checks" + ) + print(stdout) + + +def check_output(output): + return regex.match(r"^/.* warning: ", output) + + +def tidy(args): + files = util.input_files(args.files) + + groups = Multimap() + + for file in files: + groups["test" if "/tests/" in file else "main"] = file + + fix = "--fix" if args.fix == "fix" else "" + lines = ( + ("'--line-filter=" + git_changed_lines(args.commit)) + "'" + if args.commit is not None + else "" + ) + + ok = True + if groups.get("main", None): + status, stdout, stderr = util.run( + f"xargs clang-tidy -p=build/release/ --format-style=file -header-filter='.*' --checks='{CODE_CHECKS}' --quiet {fix} {lines}", + input=groups["main"], + ) + ok = check_output(stdout) and ok + + if groups.get("test", None): + status, stdout, stderr = util.run( + f"xargs clang-tidy -p=build/release/ --format-style=file -header-filter='.*' --checks='{TEST_CHECKS}' --quiet {fix} {lines}", + input=groups["test"], + ) + ok = check_output(stdout) and ok + + return 0 if ok else 1 + + +def parse_args(): + global parser + parser = argparse.ArgumentParser(description="CircliCi Utility") + parser.add_argument("--commit") + parser.add_argument("--fix") + + parser.add_argument("files", metavar="FILES", nargs="+", help="files to process") + + return parser.parse_args() + + +def main(): + return tidy(parse_args()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/setup-check.sh b/scripts/setup-check.sh new file mode 100644 index 000000000000..eb1069d6f57e --- /dev/null +++ b/scripts/setup-check.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -x + +export DEBIAN_FRONTEND=noninteractive +apt update +apt install --no-install-recommends -y clang-format-12 python3-pip git make ssh +pip3 install cmake_format black +pip3 cache purge +apt purge --auto-remove -y python3-pip +update-alternatives --install /usr/bin/clang-format clang-format "$(command -v clang-format-12)" 12 +apt clean diff --git a/scripts/setup-circleci.sh b/scripts/setup-circleci.sh new file mode 100755 index 000000000000..7c21ded3199b --- /dev/null +++ b/scripts/setup-circleci.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -efx -o pipefail +# Some of the packages must be build with the same compiler flags +# so that some low level types are the same size. Also, disable warnings. +export CFLAGS="-mavx2 -mfma -mavx -mf16c -masm=intel -mlzcnt -w -std=c++17" # Used by LZO. +export CXXFLAGS=$CFLAGS # Used by boost. +export CPPFLAGS=$CFLAGS # Used by LZO. + +function dnf_install { + dnf install -y -q --setopt=install_weak_deps=False "$@" +} + +dnf_install epel-release dnf-plugins-core # For ccache, ninja +dnf config-manager --set-enabled powertools +dnf_install ninja-build ccache gcc-toolset-9 git wget which libevent-devel \ + openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ + protobuf-devel fmt-devel + +dnf remove -y gflags + +# Required for Thrift +dnf_install autoconf automake libtool bison flex + +# Activate gcc9; enable errors on unset variables afterwards. +source /opt/rh/gcc-toolset-9/enable || exit 1 +set -u + +function cmake_install { + cmake -B "$1-build" -GNinja -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_CXX_FLAGS="${CFLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -Wno-dev "$@" + ninja -C "$1-build" install +} + +function wget_and_untar { + local URL=$1 + local DIR=$2 + mkdir -p "${DIR}" + wget -q --max-redirect 3 -O - "${URL}" | tar -xz -C "${DIR}" --strip-components=1 +} + +# untar cmake binary release directly to /usr. +wget_and_untar https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz /usr & + +# Fetch sources. +wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags & +wget_and_untar https://github.com/google/glog/archive/v0.4.0.tar.gz glog & +wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo & +wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost & +wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy & +wget_and_untar https://github.com/google/googletest/archive/release-1.10.0.tar.gz googletest & +wget_and_untar https://github.com/facebook/folly/archive/v2021.05.10.00.tar.gz folly & +# wget_and_untar https://github.com/ericniebler/range-v3/archive/0.11.0.tar.gz ranges-v3 & + +wait # For cmake and source downloads to complete. + +# Build & install. +( + cd lzo + ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10 + make "-j$(nproc)" + make install +) + +( + cd boost + ./bootstrap.sh --prefix=/usr/local + ./b2 "-j$(nproc)" -d0 install threading=multi +) + +cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr +cmake_install glog -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr +cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF +cmake_install googletest -DBUILD_SHARED_LIBS=ON +# Folly fails to build in release-mode due +# AtomicUtil-inl.h:202: Error: operand type mismatch for `bts' +cmake_install folly -DCMAKE_BUILD_TYPE=Debug +# cmake_install ranges-v3 + +dnf clean all diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh new file mode 100755 index 000000000000..a3464c450700 --- /dev/null +++ b/scripts/setup-macos.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script documents setting up a macOS host for presto_cpp +# development. Running it should make you ready to compile. +# +# Environment variables: +# * INSTALL_PREREQUISITES="N": Skip installation of brew/pip deps. +# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts. +# Use "n" to never wipe directories. +# +# You can also run individual functions below by specifying them as arguments: +# $ scripts/setup-macos.sh install_googletest install_fmt +# + +set -e # Exit on error. +set -x # Print commands that are executed. + +FB_OS_VERSION=v2021.05.10.00 +NPROC=$(sysctl -n hw.physicalcpu) +COMPILER_FLAGS="-mavx2 -mfma -mavx -mf16c -masm=intel -mlzcnt" +DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} +MACOS_DEPS="ninja cmake ccache protobuf icu4c boost double-conversion gflags glog libevent lz4 lzo snappy xz zstd" + +function run_and_time { + time "$@" + { echo "+ Finished running $*"; } 2> /dev/null +} + +function prompt { + ( + while true; do + local input="${PROMPT_ALWAYS_RESPOND}" + echo -n "$(tput bold)$* [Y, n]$(tput sgr0) " + [[ -z "${input}" ]] && read input + if [[ "${input}" == "Y" || "${input}" == "y" || "${input}" == "" ]]; then + return 0 + elif [[ "${input}" == "N" || "${input}" == "n" ]]; then + return 1 + fi + done + ) 2> /dev/null +} + +# github_checkout $REPO $VERSION clones or re-uses an existing clone of the +# specified repo, checking out the requested version. +function github_checkout { + local REPO=$1 + local VERSION=$2 + local DIRNAME=$(basename "$1") + + cd "${DEPENDENCY_DIR}" + if [ -z "${DIRNAME}" ]; then + echo "Failed to get repo name from $1" + exit 1 + fi + if [ -d "${DIRNAME}" ] && prompt "${DIRNAME} already exists. Delete?"; then + rm -rf "${DIRNAME}" + fi + if [ ! -d "${DIRNAME}" ]; then + git clone -q "https://github.com/${REPO}.git" + fi + cd "${DIRNAME}" + git fetch -q + git checkout "${VERSION}" +} + +function cmake_install { + local NAME=$(basename "$(pwd)") + local BINARY_DIR=_build + if [ -d "${BINARY_DIR}" ] && prompt "Do you want to rebuild ${NAME}?"; then + rm -rf "${BINARY_DIR}" + fi + mkdir -p "${BINARY_DIR}" + cmake -Wno-dev -B"${BINARY_DIR}" \ + -GNinja \ + -DCMAKE_CXX_STANDARD=17 \ + "${INSTALL_PREFIX+-DCMAKE_PREFIX_PATH=}${INSTALL_PREFIX-}" \ + "${INSTALL_PREFIX+-DCMAKE_INSTALL_PREFIX=}${INSTALL_PREFIX-}" \ + -DCMAKE_CXX_FLAGS="${COMPILER_FLAGS}" \ + "$@" + ninja -C "${BINARY_DIR}" install +} + +function update_brew { + /usr/local/bin/brew update --force --quiet +} + +function install_build_prerequisites { + for pkg in ${MACOS_DEPS} + do + brew install --formula $pkg && echo "Installation of $pkg is successful" || brew upgrade --formula $pkg + done + + pip3 install --user cmake-format regex +} + +function install_googletest { + github_checkout google/googletest release-1.10.0 + cmake_install +} + +function install_fmt { + github_checkout fmtlib/fmt 7.1.3 + cmake_install -DFMT_TEST=OFF +} + +function install_folly { + github_checkout facebook/folly "${FB_OS_VERSION}" + cmake_install -DBUILD_TESTS=OFF -DCMAKE_PREFIX_PATH="$(brew --prefix openssl)" +} + +function install_ranges_v3 { + github_checkout ericniebler/range-v3 master + cmake_install -DRANGES_ENABLE_WERROR=OFF +} + +function install_re2 { + github_checkout google/re2 2021-04-01 + cmake_install +} + +function install_velox_deps { + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + run_and_time install_build_prerequisites + fi + run_and_time install_ranges_v3 + run_and_time install_googletest + run_and_time install_fmt + run_and_time install_folly + run_and_time install_re2 +} + +(return 2> /dev/null) && return # If script was sourced, don't run commands. + +( + if [[ $# -ne 0 ]]; then + for cmd in "$@"; do + run_and_time "${cmd}" + done + else + install_velox_deps + fi +) diff --git a/scripts/tests/CMakeLists.txt b/scripts/tests/CMakeLists.txt new file mode 100644 index 000000000000..7a806427823c --- /dev/null +++ b/scripts/tests/CMakeLists.txt @@ -0,0 +1,12 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +add_test(test_LicenseHeader ${CMAKE_CURRENT_SOURCE_DIR}/test_LicenseHeader.sh) diff --git a/scripts/tests/TestFramework.sh b/scripts/tests/TestFramework.sh new file mode 100644 index 000000000000..6fb3bc53b392 --- /dev/null +++ b/scripts/tests/TestFramework.sh @@ -0,0 +1,174 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TestsRun=0 +TestsPassed=0 +TestsFailed=0 +ExitOnFail=0 +KeepTest=0 + +while [ $# -gt 0 ] ; do + case $1 in + -help) +cat 1>&2 << EOF + -c cleanup + -k keep temps + -e exit on failure + -x shell debug +EOF + exit ;; + -c) # Clean Up + # + rm -f *.tmp + exit 0 + ;; + -k) KeepTest=1 ;; + -e) ExitOnFail=1 ;; + -x) SetMinusX=1 ;; + *) Select=$1;; + esac + shift +done + +printf -- "Number %-52s Result Time\n" Test +printf -- "------ %-52s ------ ------\n" ---- + +Test() { + Expect=Pass + if [ x"$1" = "x!" ] ; then Expect=Fail; shift + fi + + Title=$1; shift + TestsRun=`expr $TestsRun + 1` + + printf "%6d %-52s" $TestsRun "$Title" 1>&2 + + Start=`Clock` + if [ "$SetMinusX" = 1 ] ; then + set -x + fi +} + +Clock() { + echo $SECONDS +} + +Calc() { + awk "BEGIN { print $* }" +} + +TestCleanUp() { + local result="$1" + Now=`Clock`; Elapse=$(Calc $Now - $Start) + + if [ "$result" = Pass ] ; then TestsPassed=`expr $TestsPassed + 1` + else TestsFailed=`expr $TestsFailed + 1` + fi + printf " $result %7.3f\n" $Elapse 1>&2 + + if [ "$result" = Fail -a $ExitOnFail = 1 ] ; then + exit 1 + fi +} + +Fail() { TestCleanUp Fail; } +Pass() { TestCleanUp Pass; } + +CheckOutput() { + local x="$1" + local y="$2" + + if [ "$Expect" = Pass -a "$x" = "$y" ] ; then + return 0 + fi + if [ "$Expect" = Pass -a "$x" != "$y" ] ; then + ShowComparison "$x" "$y" + return 1 + fi + if [ "$Expect" = Fail -a "$x" != "$y" ] ; then + return 0 + fi + if [ "$Expect" = Fail -a "$x" = "$y" ] ; then + ShowComparison "$x" "$y" + return 1 + fi + + return 1 +} + +ShowComparison() { + local x="$1" + local y="$2" + + echo 1>&2 + echo ":$x:" 1>&2 + echo ":$y:" 1>&2 +} + +CompareArgs() { + while [ $# -ge 2 ] ; do + x=$1 + y=$2 + shift; shift + + if ! CheckOutput "$x" "$y" ; then + Fail; return 1 + fi + done + + Pass; return 0 +} + +CompareEval() { + while [ $# -ge 2 ] ; do + x=`$1` + y=$2 + shift; shift + + if [ "$x" != "$y" ] ; then + echo > /dev/tty + echo ":$x:" > /dev/tty + echo ":$y:" > /dev/tty + + Fail; return 1 + fi + done + + Pass; return 0 +} + +CompareFiles() { + if cmp "$1" "$2" ; then Pass; return 0 + else Fail; return 1 + fi +} + +DiffFiles() { + while [ $# != 0 ] ; do + if diff "$1" "$2" ; then Pass; return 0 + else Fail; return 1 + fi + shift + shift + done +} + +TestDone() { + if [ $KeepTest = 0 ] ; then + rm -f *.tmp *.tmp.* + fi + + echo + echo "Failed $TestsFailed" + echo "Passed $TestsPassed of $TestsRun tests run." + exit 0 +} diff --git a/scripts/tests/data/CMakeLists.expected.txt b/scripts/tests/data/CMakeLists.expected.txt new file mode 100644 index 000000000000..62abf26e3d08 --- /dev/null +++ b/scripts/tests/data/CMakeLists.expected.txt @@ -0,0 +1,48 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +cmake_minimum_required(VERSION 3.10) + +# set the project name +project(PrestoCpp) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -mavx2 -mfma -mavx -mf16c -masm=intel -mlzcnt") + +set(Boost_USE_MULTITHREADED TRUE) +find_package(Boost 1.72.0 REQUIRED program_options context filesystem regex thread system date_time atomic) +include_directories(${Boost_INCLUDE_DIRS}) + +find_package(GTest REQUIRED) +find_package(gflags COMPONENTS shared) +find_library(GLOG glog) +find_library(FOLLY folly) +include_directories(${GTEST_INCLUDE_DIRS}) + +find_package(Arrow REQUIRED) +include_directories(${ARROW_INCLUDE_DIRS}) + +find_package(double-conversion REQUIRED) + +find_library(FMT fmt) + +find_package(folly REQUIRED) +include_directories(${FOLLY_INCLUDE_DIRS}) + +include_directories(third_party) +include_directories(src) + +include(CTest) # include after project() but before add_subdirectory() + +add_subdirectory(scripts) +add_subdirectory(src) diff --git a/scripts/tests/data/CMakeLists.txt b/scripts/tests/data/CMakeLists.txt new file mode 100644 index 000000000000..aeadffaf99f0 --- /dev/null +++ b/scripts/tests/data/CMakeLists.txt @@ -0,0 +1,37 @@ +cmake_minimum_required(VERSION 3.10) + +# set the project name +project(PrestoCpp) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -mavx2 -mfma -mavx -mf16c -masm=intel -mlzcnt") + +set(Boost_USE_MULTITHREADED TRUE) +find_package(Boost 1.72.0 REQUIRED program_options context filesystem regex thread system date_time atomic) +include_directories(${Boost_INCLUDE_DIRS}) + +find_package(GTest REQUIRED) +find_package(gflags COMPONENTS shared) +find_library(GLOG glog) +find_library(FOLLY folly) +include_directories(${GTEST_INCLUDE_DIRS}) + +find_package(Arrow REQUIRED) +include_directories(${ARROW_INCLUDE_DIRS}) + +find_package(double-conversion REQUIRED) + +find_library(FMT fmt) + +find_package(folly REQUIRED) +include_directories(${FOLLY_INCLUDE_DIRS}) + +include_directories(third_party) +include_directories(src) + +include(CTest) # include after project() but before add_subdirectory() + +add_subdirectory(scripts) +add_subdirectory(src) diff --git a/scripts/tests/data/foo.almost.cpp b/scripts/tests/data/foo.almost.cpp new file mode 100644 index 000000000000..2804910f6e73 --- /dev/null +++ b/scripts/tests/data/foo.almost.cpp @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import argparse import fnmatch import os import regex import sys + + class attrdict(dict) : __getattr__ = dict.__getitem__ __setattr__ = dict.__setitem__ + + def parse_args() : parser = argparse.ArgumentParser(description = 'Update license headers') parser.add_argument('--header', default = 'license.header', help = 'header file') parser.add_argument('--extra', default = 30, help = 'extra characters past beginning of file to look for header') parser.add_argument('--editdist', default = 7, help = 'max edit distance between headers') parser.add_argument('--remove', default = False, action = "store_true", help = 'remove the header') parser.add_argument('--cslash', default = False, action = "store_true", help = 'use C slash "//" style comments') parser.add_argument('-v', default = False, action = "store_true", dest = "verbose", help = 'verbose output') + + group = parser.add_mutually_exclusive_group() group.add_argument('-k', default = False, action = "store_true", dest = "check", help = 'check headers') group.add_argument('-i', default = False, action = "store_true", dest = "inplace", help = 'edit file inplace') + + parser.add_argument('files', metavar = 'FILES', nargs = '+', help = 'files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/foo.almost.sh b/scripts/tests/data/foo.almost.sh new file mode 100644 index 000000000000..9c236996e82a --- /dev/null +++ b/scripts/tests/data/foo.almost.sh @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/foo.cpp b/scripts/tests/data/foo.cpp new file mode 100644 index 000000000000..f4f78dfbffbe --- /dev/null +++ b/scripts/tests/data/foo.cpp @@ -0,0 +1,11 @@ +import argparse import fnmatch import os import regex import sys + + class attrdict(dict) : __getattr__ = dict.__getitem__ __setattr__ = dict.__setitem__ + + def parse_args() : parser = argparse.ArgumentParser(description = 'Update license headers') parser.add_argument('--header', default = 'license.header', help = 'header file') parser.add_argument('--extra', default = 30, help = 'extra characters past beginning of file to look for header') parser.add_argument('--editdist', default = 7, help = 'max edit distance between headers') parser.add_argument('--remove', default = False, action = "store_true", help = 'remove the header') parser.add_argument('--cslash', default = False, action = "store_true", help = 'use C slash "//" style comments') parser.add_argument('-v', default = False, action = "store_true", dest = "verbose", help = 'verbose output') + + group = parser.add_mutually_exclusive_group() group.add_argument('-k', default = False, action = "store_true", dest = "check", help = 'check headers') group.add_argument('-i', default = False, action = "store_true", dest = "inplace", help = 'edit file inplace') + + parser.add_argument('files', metavar = 'FILES', nargs = '+', help = 'files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/foo.expected.cpp b/scripts/tests/data/foo.expected.cpp new file mode 100644 index 000000000000..e3f75b39c6a5 --- /dev/null +++ b/scripts/tests/data/foo.expected.cpp @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import argparse import fnmatch import os import regex import sys + + class attrdict(dict) : __getattr__ = dict.__getitem__ __setattr__ = dict.__setitem__ + + def parse_args() : parser = argparse.ArgumentParser(description = 'Update license headers') parser.add_argument('--header', default = 'license.header', help = 'header file') parser.add_argument('--extra', default = 30, help = 'extra characters past beginning of file to look for header') parser.add_argument('--editdist', default = 7, help = 'max edit distance between headers') parser.add_argument('--remove', default = False, action = "store_true", help = 'remove the header') parser.add_argument('--cslash', default = False, action = "store_true", help = 'use C slash "//" style comments') parser.add_argument('-v', default = False, action = "store_true", dest = "verbose", help = 'verbose output') + + group = parser.add_mutually_exclusive_group() group.add_argument('-k', default = False, action = "store_true", dest = "check", help = 'check headers') group.add_argument('-i', default = False, action = "store_true", dest = "inplace", help = 'edit file inplace') + + parser.add_argument('files', metavar = 'FILES', nargs = '+', help = 'files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/foo.expected.h b/scripts/tests/data/foo.expected.h new file mode 100644 index 000000000000..e3f75b39c6a5 --- /dev/null +++ b/scripts/tests/data/foo.expected.h @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import argparse import fnmatch import os import regex import sys + + class attrdict(dict) : __getattr__ = dict.__getitem__ __setattr__ = dict.__setitem__ + + def parse_args() : parser = argparse.ArgumentParser(description = 'Update license headers') parser.add_argument('--header', default = 'license.header', help = 'header file') parser.add_argument('--extra', default = 30, help = 'extra characters past beginning of file to look for header') parser.add_argument('--editdist', default = 7, help = 'max edit distance between headers') parser.add_argument('--remove', default = False, action = "store_true", help = 'remove the header') parser.add_argument('--cslash', default = False, action = "store_true", help = 'use C slash "//" style comments') parser.add_argument('-v', default = False, action = "store_true", dest = "verbose", help = 'verbose output') + + group = parser.add_mutually_exclusive_group() group.add_argument('-k', default = False, action = "store_true", dest = "check", help = 'check headers') group.add_argument('-i', default = False, action = "store_true", dest = "inplace", help = 'edit file inplace') + + parser.add_argument('files', metavar = 'FILES', nargs = '+', help = 'files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/foo.expected.py b/scripts/tests/data/foo.expected.py new file mode 100644 index 000000000000..f3acaa9abfb8 --- /dev/null +++ b/scripts/tests/data/foo.expected.py @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() + + diff --git a/scripts/tests/data/foo.expected.sh b/scripts/tests/data/foo.expected.sh new file mode 100644 index 000000000000..1492c4160083 --- /dev/null +++ b/scripts/tests/data/foo.expected.sh @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/foo.h b/scripts/tests/data/foo.h new file mode 100644 index 000000000000..f4f78dfbffbe --- /dev/null +++ b/scripts/tests/data/foo.h @@ -0,0 +1,11 @@ +import argparse import fnmatch import os import regex import sys + + class attrdict(dict) : __getattr__ = dict.__getitem__ __setattr__ = dict.__setitem__ + + def parse_args() : parser = argparse.ArgumentParser(description = 'Update license headers') parser.add_argument('--header', default = 'license.header', help = 'header file') parser.add_argument('--extra', default = 30, help = 'extra characters past beginning of file to look for header') parser.add_argument('--editdist', default = 7, help = 'max edit distance between headers') parser.add_argument('--remove', default = False, action = "store_true", help = 'remove the header') parser.add_argument('--cslash', default = False, action = "store_true", help = 'use C slash "//" style comments') parser.add_argument('-v', default = False, action = "store_true", dest = "verbose", help = 'verbose output') + + group = parser.add_mutually_exclusive_group() group.add_argument('-k', default = False, action = "store_true", dest = "check", help = 'check headers') group.add_argument('-i', default = False, action = "store_true", dest = "inplace", help = 'edit file inplace') + + parser.add_argument('files', metavar = 'FILES', nargs = '+', help = 'files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/foo.py b/scripts/tests/data/foo.py new file mode 100644 index 000000000000..057c686c3504 --- /dev/null +++ b/scripts/tests/data/foo.py @@ -0,0 +1,35 @@ +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() + + diff --git a/scripts/tests/data/foo.sh b/scripts/tests/data/foo.sh new file mode 100644 index 000000000000..90b4c800df35 --- /dev/null +++ b/scripts/tests/data/foo.sh @@ -0,0 +1,33 @@ +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/hashbang.almost.sh b/scripts/tests/data/hashbang.almost.sh new file mode 100644 index 000000000000..22702ab539d1 --- /dev/null +++ b/scripts/tests/data/hashbang.almost.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/hashbang.expected.py b/scripts/tests/data/hashbang.expected.py new file mode 100644 index 000000000000..6c164da0f5bf --- /dev/null +++ b/scripts/tests/data/hashbang.expected.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() + diff --git a/scripts/tests/data/hashbang.expected.sh b/scripts/tests/data/hashbang.expected.sh new file mode 100644 index 000000000000..22702ab539d1 --- /dev/null +++ b/scripts/tests/data/hashbang.expected.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() diff --git a/scripts/tests/data/hashbang.py b/scripts/tests/data/hashbang.py new file mode 100644 index 000000000000..83b5326d1ce7 --- /dev/null +++ b/scripts/tests/data/hashbang.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() + diff --git a/scripts/tests/data/hashbang.sh b/scripts/tests/data/hashbang.sh new file mode 100644 index 000000000000..1b3da8e616bf --- /dev/null +++ b/scripts/tests/data/hashbang.sh @@ -0,0 +1,34 @@ +#!/bin/bash +import argparse +import fnmatch +import os +import regex +import sys + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + +def parse_args(): + parser = argparse.ArgumentParser(description='Update license headers') + parser.add_argument('--header', default='license.header', help='header file') + parser.add_argument('--extra', default=30, + help='extra characters past beginning of file to look for header') + parser.add_argument('--editdist', default=7, help='max edit distance between headers') + parser.add_argument('--remove', default=False, action="store_true", + help='remove the header') + parser.add_argument('--cslash', default=False, action="store_true", + help='use C slash "//" style comments') + parser.add_argument('-v', default=False, action="store_true", dest="verbose", + help='verbose output') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-k', default=False, action="store_true", dest="check", + help='check headers') + group.add_argument('-i', default=False, action="store_true", dest="inplace", + help='edit file inplace') + + parser.add_argument('files', metavar='FILES', nargs='+', + help='files to process') + + return parser.parse_args() diff --git a/scripts/tests/test_LicenseHeader.sh b/scripts/tests/test_LicenseHeader.sh new file mode 100755 index 000000000000..b7365088a480 --- /dev/null +++ b/scripts/tests/test_LicenseHeader.sh @@ -0,0 +1,111 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRC=$(dirname $0) +DATA=$SRC/data + +source $SRC/TestFramework.sh + +LICENSE_HEADER=$SRC/../license-header.py +LICENSE_HEADER_FILE=$SRC/../../license.header + +license_header() { + $SRC/../license-header.py --header $LICENSE_HEADER_FILE "$@" +} + +TestFile() { + local title="$1" ; shift + local file="$1" ; shift + local expected="$1" ; shift + + Test "license-header.py $title file $file" + cp $DATA/$file . + license_header "$@" -i $file + + DiffFiles $file $DATA/$expected +} + +# Test header insertion +# +TestFile Insert foo.cpp foo.expected.cpp +TestFile Insert foo.h foo.expected.h +TestFile Insert foo.sh foo.expected.sh +TestFile Insert foo.py foo.expected.py +TestFile Insert hashbang.sh hashbang.expected.sh +TestFile Insert hashbang.py hashbang.expected.py +TestFile Insert CMakeLists.txt CMakeLists.expected.txt + +# Test header found - don't change file further +# +TestFile Again foo.expected.cpp foo.expected.cpp +TestFile Again foo.expected.h foo.expected.h +TestFile Again foo.expected.sh foo.expected.sh +TestFile Again foo.expected.py foo.expected.py +TestFile Again hashbang.expected.sh hashbang.expected.sh +TestFile Again hashbang.expected.py hashbang.expected.py + +TestFile Remove foo.expected.cpp foo.cpp --remove +TestFile Remove foo.expected.h foo.h --remove +TestFile Remove foo.expected.sh foo.sh --remove +TestFile Remove foo.expected.py foo.py --remove +TestFile Remove hashbang.expected.sh hashbang.sh --remove +TestFile Remove hashbang.expected.py hashbang.py --remove + +# Test header found - Close match +# +TestFile Almost foo.almost.cpp foo.expected.cpp +TestFile Almost foo.almost.sh foo.expected.sh +TestFile Almost hashbang.almost.sh hashbang.expected.sh + +Test "List of Files in stdin" + cp $DATA/foo.sh . + + echo foo.sh | license_header -i - + + DiffFiles foo.sh $DATA/foo.expected.sh + +Test "Header Check Only - OK" + cp $DATA/foo.expected.sh . + cp $DATA/foo.expected.cpp . + + if license_header "$@" -k foo.expected.sh foo.expected.cpp ; then + Pass + else + Fail + fi + +Test "Header Check Only - Fix" + cp $DATA/foo.sh . + cp $DATA/foo.cpp . + + if license_header "$@" -k foo.sh foo.cpp; then + Fail + else + Pass + fi + +Test "Header Check Verbose" + cp $DATA/foo.sh . + cp $DATA/foo.expected.sh . + cp $DATA/foo.cpp . + cp $DATA/foo.expected.cpp . + + result=$(license_header "$@" -vk foo.sh foo.expected.sh foo.cpp foo.expected.cpp) + expected="\ +Fix : foo.sh +OK : foo.expected.sh +Fix : foo.cpp +OK : foo.expected.cpp" + + CompareArgs "$result" "$expected" + +TestDone diff --git a/scripts/util.py b/scripts/util.py new file mode 100644 index 000000000000..e2740fcba418 --- /dev/null +++ b/scripts/util.py @@ -0,0 +1,87 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import json +import os +import regex +import subprocess +import sys + + +class attrdict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + + +class string(str): + def extract(self, rexp): + return regex.match(rexp, self).group(1) + + def json(self): + return json.loads(self, object_hook=attrdict) + + +def run(command, compressed=False, **kwargs): + if "input" in kwargs: + input = kwargs["input"] + + if type(input) == list: + input = "\n".join(input) + "\n" + + kwargs["input"] = input.encode("utf-8") + + reply = subprocess.run( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) + + if compressed: + stdout = gzip.decompress(reply.stdout) + else: + stdout = reply.stdout + + stdout = ( + string(stdout.decode("utf-8", errors="ignore").strip()) + if stdout is not None + else "" + ) + stderr = ( + string(reply.stderr.decode("utf-8").strip()) if reply.stderr is not None else "" + ) + + if stderr != "": + print(stderr, file=sys.stderr) + + return reply.returncode, stdout, stderr + + +def get_filename(filename): + return os.path.basename(filename) + + +def get_fileextn(filename): + split = os.path.splitext(filename) + if len(split) <= 1: + return "" + + return split[-1] + + +def script_path(): + return os.path.dirname(os.path.realpath(sys.argv[0])) + + +def input_files(files): + if len(files) == 1 and files[0] == "-": + return [file.strip() for file in sys.stdin.readlines()] + else: + return files diff --git a/velox/CMakeLists.txt b/velox/CMakeLists.txt new file mode 100644 index 000000000000..6c97f831e237 --- /dev/null +++ b/velox/CMakeLists.txt @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +add_subdirectory(aggregates) +add_subdirectory(buffer) +add_subdirectory(common) +add_subdirectory(duckdb) +add_subdirectory(connectors) +add_subdirectory(core) +add_subdirectory(dwio) +add_subdirectory(exec) +add_subdirectory(expression) +add_subdirectory(external/date) +add_subdirectory(external/duckdb) +add_subdirectory(external/md5) +add_subdirectory(functions) +add_subdirectory(parse) +add_subdirectory(serializers) +add_subdirectory(type) +add_subdirectory(vector) +add_subdirectory(row) +add_subdirectory(flag_definitions) +add_subdirectory(codegen) +if(${CODEGEN_SUPPORT}) + add_subdirectory(experimental/codegen) +endif() diff --git a/velox/aggregates/AggregateNames.h b/velox/aggregates/AggregateNames.h new file mode 100644 index 000000000000..504a0ffe887b --- /dev/null +++ b/velox/aggregates/AggregateNames.h @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace facebook::velox::aggregate { + +const char* const kApproxDistinct = "approx_distinct"; +const char* const kApproxPercentile = "approx_percentile"; +const char* const kArbitrary = "arbitrary"; +const char* const kArrayAgg = "array_agg"; +const char* const kAvg = "avg"; +const char* const kBitwiseAnd = "bitwise_and_agg"; +const char* const kBitwiseOr = "bitwise_or_agg"; +const char* const kBoolAnd = "bool_and"; +const char* const kBoolOr = "bool_or"; +const char* const kCount = "count"; +const char* const kCountIf = "count_if"; +const char* const kMapAgg = "map_agg"; +const char* const kMax = "max"; +const char* const kMaxBy = "max_by"; +const char* const kMin = "min"; +const char* const kMinBy = "min_by"; +const char* const kSum = "sum"; + +} // namespace facebook::velox::aggregate diff --git a/velox/aggregates/AggregationHook.h b/velox/aggregates/AggregationHook.h new file mode 100644 index 000000000000..fbfb0f87e388 --- /dev/null +++ b/velox/aggregates/AggregationHook.h @@ -0,0 +1,210 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "velox/common/base/Range.h" +#include "velox/vector/LazyVector.h" + +namespace facebook::velox::aggregate { + +class AggregationHook : public ValueHook { + public: + // Constants for identifying hooks for specialized template instantiations. + + static constexpr Kind kSumFloatToDouble = 1; + static constexpr Kind kSumDoubleToDouble = 2; + static constexpr Kind kSumIntegerToBigint = 3; + static constexpr Kind kSumBigintToBigint = 4; + static constexpr Kind kBigintMax = 5; + static constexpr Kind kBigintMin = 6; + static constexpr Kind kFloatMax = 7; + static constexpr Kind kFloatMin = 8; + static constexpr Kind kDoubleMax = 9; + static constexpr Kind kDoubleMin = 10; + + // Make null behavior known at compile time. This is useful when + // templating a column decoding loop with a hook. + static constexpr bool kSkipNulls = true; + + AggregationHook( + int32_t offset, + int32_t nullByte, + uint8_t nullMask, + char** groups, + uint64_t* numNulls) + : offset_(offset), + nullByte_(nullByte), + nullMask_(nullMask), + clearNullMask_(~nullMask_), + groups_(groups), + numNulls_(numNulls) {} + + bool acceptsNulls() const override final { + return false; + } + + // Fallback implementation of fast path. Prefer defining special + // cases for all subclasses. + void addValues( + const vector_size_t* rows, + const void* values, + vector_size_t size, + uint8_t valueWidth) override { + auto valuesAsChar = reinterpret_cast(values); + for (auto i = 0; i < size; ++i) { + addValue(rows[i], valuesAsChar + valueWidth * i); + } + } + + protected: + inline char* findGroup(vector_size_t row) { + return groups_[row]; + } + + inline bool clearNull(char* group) { + if (*numNulls_) { + uint8_t mask = group[nullByte_]; + if (mask & nullMask_) { + group[nullByte_] = mask & clearNullMask_; + --*numNulls_; + return true; + } + } + return false; + } + + int32_t currentRow_ = 0; + const int32_t offset_; + const int32_t nullByte_; + const uint8_t nullMask_; + const uint8_t clearNullMask_; + char* const* const groups_; + uint64_t* numNulls_; +}; + +template +class SumHook final : public AggregationHook { + public: + SumHook( + int32_t offset, + int32_t nullByte, + uint8_t nullMask, + char** groups, + uint64_t* numNulls) + : AggregationHook(offset, nullByte, nullMask, groups, numNulls) {} + + Kind kind() const override { + if (std::is_same::value) { + if (std::is_same::value) { + return kSumDoubleToDouble; + } + if (std::is_same::value) { + return kSumFloatToDouble; + } + } else if (std::is_same::value) { + if (std::is_same::value) { + return kSumIntegerToBigint; + } + if (std::is_same::value) { + return kSumBigintToBigint; + } + } + return kGeneric; + } + + void addValue(vector_size_t row, const void* value) override { + auto group = findGroup(row); + clearNull(group); + *reinterpret_cast(group + offset_) += + *reinterpret_cast(value); + } +}; + +template +class SimpleCallableHook final : public AggregationHook { + public: + SimpleCallableHook( + int32_t offset, + int32_t nullByte, + uint8_t nullMask, + char** groups, + uint64_t* numNulls, + UpdateSingleValue updateSingleValue) + : AggregationHook(offset, nullByte, nullMask, groups, numNulls), + updateSingleValue_(updateSingleValue) {} + + Kind kind() const override { + return kGeneric; + } + + void addValue(vector_size_t row, const void* value) override { + auto group = findGroup(row); + clearNull(group); + updateSingleValue_( + *reinterpret_cast(group + offset_), + *reinterpret_cast(value)); + } + + private: + UpdateSingleValue updateSingleValue_; +}; + +template +class MinMaxHook final : public AggregationHook { + public: + MinMaxHook( + int32_t offset, + int32_t nullByte, + uint8_t nullMask, + char** groups, + uint64_t* numNulls) + : AggregationHook(offset, nullByte, nullMask, groups, numNulls) {} + + Kind kind() const override { + if (isMin) { + if (std::is_same::value) { + return kBigintMin; + } + if (std::is_same::value) { + return kFloatMin; + } + if (std::is_same::value) { + return kDoubleMin; + } + } else { + if (std::is_same::value) { + return kBigintMax; + } + if (std::is_same::value) { + return kFloatMax; + } + if (std::is_same::value) { + return kDoubleMax; + } + } + return kGeneric; + } + + void addValue(vector_size_t row, const void* value) override { + auto group = findGroup(row); + if (clearNull(group) || + (*reinterpret_cast(group + offset_) > + *reinterpret_cast(value)) == isMin) { + *reinterpret_cast(group + offset_) = + *reinterpret_cast(value); + } + } +}; + +} // namespace facebook::velox::aggregate diff --git a/velox/aggregates/ApproxDistinct.cpp b/velox/aggregates/ApproxDistinct.cpp new file mode 100644 index 000000000000..5633662c3bdb --- /dev/null +++ b/velox/aggregates/ApproxDistinct.cpp @@ -0,0 +1,398 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define XXH_INLINE_ALL +#include "velox/aggregates/AggregateNames.h" +#include "velox/aggregates/hyperloglog/DenseHll.h" +#include "velox/aggregates/hyperloglog/HllUtils.h" +#include "velox/aggregates/hyperloglog/SparseHll.h" +#include "velox/exec/Aggregate.h" +#include "velox/exec/HashStringAllocator.h" +#include "velox/external/xxhash.h" +#include "velox/vector/DecodedVector.h" +#include "velox/vector/FlatVector.h" + +namespace facebook::velox::aggregate { +namespace { + +struct HllAccumulator { + explicit HllAccumulator(exec::HashStringAllocator* allocator) + : sparseHll_{allocator}, denseHll_{allocator} {} + + void setIndexBitLength(int8_t indexBitLength) { + indexBitLength_ = indexBitLength; + sparseHll_.setSoftMemoryLimit( + hll::DenseHll::estimateInMemorySize(indexBitLength_)); + } + + void append(uint64_t hash) { + if (isSparse_) { + if (sparseHll_.insertHash(hash)) { + toDense(); + } + } else { + denseHll_.insertHash(hash); + } + } + + int64_t cardinality() const { + return isSparse_ ? sparseHll_.cardinality() : denseHll_.cardinality(); + } + + void mergeWith(StringView serialized, exec::HashStringAllocator* allocator) { + auto input = serialized.data(); + if (hll::SparseHll::canDeserialize(input)) { + if (isSparse_) { + sparseHll_.mergeWith(input); + } else { + hll::SparseHll other{input, allocator}; + other.toDense(denseHll_); + } + } else if (hll::DenseHll::canDeserialize(input)) { + if (isSparse_) { + if (indexBitLength_ < 0) { + setIndexBitLength(hll::DenseHll::deserializeIndexBitLength(input)); + } + toDense(); + } + denseHll_.mergeWith(input); + } else { + VELOX_UNREACHABLE("Unexpected type of HLL"); + } + } + + int32_t serializedSize() { + return isSparse_ ? sparseHll_.serializedSize() : denseHll_.serializedSize(); + } + + void serialize(int8_t indexBitLength, StringView& output) { + char* outputBuffer = const_cast(output.data()); + return isSparse_ ? sparseHll_.serialize(indexBitLength, outputBuffer) + : denseHll_.serialize(outputBuffer); + } + + void toDense() { + isSparse_ = false; + denseHll_.initialize(indexBitLength_); + sparseHll_.toDense(denseHll_); + sparseHll_.reset(); + } + + bool isSparse_{true}; + int8_t indexBitLength_{-1}; + hll::SparseHll sparseHll_; + hll::DenseHll denseHll_; +}; + +template +inline uint64_t hashOne(T value) { + return XXH64(&value, sizeof(T), 0); +} + +template <> +inline uint64_t hashOne(StringView value) { + return XXH64(value.data(), value.size(), 0); +} + +template +class ApproxDistinctAggregate : public exec::Aggregate { + public: + ApproxDistinctAggregate( + core::AggregationNode::Step step, + const TypePtr& resultType) + : exec::Aggregate(step, resultType) {} + + int32_t accumulatorFixedWidthSize() const override { + return sizeof(HllAccumulator); + } + + void initializeNewGroups( + char** groups, + folly::Range indices) override { + setAllNulls(groups, indices); + for (auto i : indices) { + auto group = groups[i]; + new (group + offset_) HllAccumulator(allocator_); + } + } + + void finalize(char** /*groups*/, int32_t /*numGroups*/) override { + // nothing to do + } + + void extractValues(char** groups, int32_t numGroups, VectorPtr* result) + override { + VELOX_CHECK(result); + auto flatResult = (*result)->asFlatVector(); + + extract( + groups, + numGroups, + flatResult, + [](HllAccumulator* accumulator, + FlatVector* result, + vector_size_t index) { + result->set(index, accumulator->cardinality()); + }); + } + + void extractAccumulators(char** groups, int32_t numGroups, VectorPtr* result) + override { + VELOX_CHECK(result); + auto flatResult = (*result)->asFlatVector(); + + extract( + groups, + numGroups, + flatResult, + [&](HllAccumulator* accumulator, + FlatVector* result, + vector_size_t index) { + auto size = accumulator->serializedSize(); + if (StringView::isInline(size)) { + StringView serialized(size); + accumulator->serialize(indexBitLength_, serialized); + result->setNoCopy(index, serialized); + } else { + Buffer* buffer = flatResult->getBufferWithSpace(size); + StringView serialized(buffer->as() + buffer->size(), size); + accumulator->serialize(indexBitLength_, serialized); + buffer->setSize(buffer->size() + size); + result->setNoCopy(index, serialized); + } + }); + } + + void destroy(folly::Range /*groups*/) override {} + + protected: + void updatePartial( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodeArguments(rows, args); + + rows.applyToSelected([&](auto row) { + if (decodedValue_.isNullAt(row)) { + return; + } + + auto group = groups[row]; + auto accumulator = value(group); + if (clearNull(group)) { + accumulator->setIndexBitLength(indexBitLength_); + } + + auto hash = hashOne(decodedValue_.valueAt(row)); + accumulator->append(hash); + }); + } + + void updateFinal( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodedHll_.decode(*args[0], rows, true); + + rows.applyToSelected([&](auto row) { + if (decodedHll_.isNullAt(row)) { + return; + } + + auto group = groups[row]; + clearNull(group); + + auto serialized = decodedHll_.valueAt(row); + + auto accumulator = value(group); + accumulator->mergeWith(serialized, allocator_); + }); + } + + void updateSingleGroupPartial( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodeArguments(allRows, args); + + allRows.applyToSelected([&](auto row) { + if (decodedValue_.isNullAt(row)) { + return; + } + + auto accumulator = value(group); + if (clearNull(group)) { + accumulator->setIndexBitLength(indexBitLength_); + } + + auto hash = hashOne(decodedValue_.valueAt(row)); + accumulator->append(hash); + }); + } + + void updateSingleGroupFinal( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodedHll_.decode(*args[0], allRows, true); + + allRows.applyToSelected([&](auto row) { + if (decodedHll_.isNullAt(row)) { + return; + } + + clearNull(group); + + auto serialized = decodedHll_.valueAt(row); + + auto accumulator = value(group); + accumulator->mergeWith(serialized, allocator_); + }); + } + + private: + template + void extract( + char** groups, + int32_t numGroups, + FlatVector* result, + ExtractFunc extractFunction) { + VELOX_CHECK(result); + result->resize(numGroups); + + uint64_t* rawNulls = nullptr; + if (result->mayHaveNulls()) { + BufferPtr nulls = result->mutableNulls(result->size()); + rawNulls = nulls->asMutable(); + } + + for (auto i = 0; i < numGroups; ++i) { + char* group = groups[i]; + if (isNull(group)) { + result->setNull(i, true); + } else { + if (rawNulls) { + bits::clearBit(rawNulls, i); + } + + auto accumulator = value(group); + extractFunction(accumulator, result, i); + } + } + } + + void decodeArguments( + const SelectivityVector& rows, + const std::vector& args) { + decodedValue_.decode(*args[0], rows, true); + if (args.size() > 1) { + decodedMaxStandardError_.decode(*args[1], rows, true); + checkSetMaxStandardError(); + } + } + + void checkSetMaxStandardError() { + VELOX_CHECK( + decodedMaxStandardError_.isConstantMapping(), + "Max standard error argument must be constant for all input rows"); + + auto maxStandardError = decodedMaxStandardError_.valueAt(0); + checkSetMaxStandardError(maxStandardError); + } + + void checkSetMaxStandardError(double error) { + VELOX_USER_CHECK_GE( + error, + hll::kLowestMaxStandardError, + "Max standard error must be in [{}, {}] range", + hll::kLowestMaxStandardError, + hll::kHighestMaxStandardError); + VELOX_USER_CHECK_LE( + error, + hll::kHighestMaxStandardError, + "Max standard error must be in [{}, {}] range", + hll::kLowestMaxStandardError, + hll::kHighestMaxStandardError); + + if (maxStandardError_ < 0) { + maxStandardError_ = error; + indexBitLength_ = hll::toIndexBitLength(error); + } else { + VELOX_USER_CHECK_EQ( + error, + maxStandardError_, + "Max standard error argument must be constant for all input rows"); + } + } + + int8_t indexBitLength_{hll::toIndexBitLength(hll::kDefaultStandardError)}; + double maxStandardError_{-1}; + DecodedVector decodedValue_; + DecodedVector decodedMaxStandardError_; + DecodedVector decodedHll_; +}; + +template +std::unique_ptr createApproxDistinct( + core::AggregationNode::Step step, + const TypePtr& resultType) { + using T = typename TypeTraits::NativeType; + return std::make_unique>(step, resultType); +} + +bool registerApproxDistinct(const std::string& name) { + exec::AggregateFunctions().Register( + name, + [name]( + core::AggregationNode::Step step, + const std::vector& argTypes, + const TypePtr& /*resultType*/) -> std::unique_ptr { + auto isRawInput = exec::isRawInput(step); + auto isPartialOutput = exec::isPartialOutput(step); + + if (isRawInput) { + VELOX_USER_CHECK_GE( + argTypes.size(), 1, "{} takes 1 or 2 arguments", name); + VELOX_USER_CHECK_LE( + argTypes.size(), 2, "{} takes 1 or 2 arguments", name); + } else { + VELOX_USER_CHECK_EQ( + argTypes.size(), + 1, + "The type of partial result for {} must be VARBINARY", + name); + VELOX_USER_CHECK_GE( + argTypes[0]->kind(), + TypeKind::VARBINARY, + "The type of partial result for {} must be VARBINARY", + name); + } + + TypePtr type = isRawInput ? argTypes[0] : BIGINT(); + TypePtr aggResultType = isPartialOutput + ? std::dynamic_pointer_cast(VARBINARY()) + : BIGINT(); + return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH( + createApproxDistinct, type->kind(), step, aggResultType); + }); + return true; +} + +static bool FB_ANONYMOUS_VARIABLE(g_AggregateFunction) = + registerApproxDistinct(kApproxDistinct); +} // namespace +} // namespace facebook::velox::aggregate diff --git a/velox/aggregates/ApproxPercentile.cpp b/velox/aggregates/ApproxPercentile.cpp new file mode 100644 index 000000000000..eba664dfc197 --- /dev/null +++ b/velox/aggregates/ApproxPercentile.cpp @@ -0,0 +1,580 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "velox/aggregates/AggregateNames.h" +#include "velox/aggregates/IOUtils.h" +#include "velox/exec/Aggregate.h" +#include "velox/exec/HashStringAllocator.h" +#include "velox/vector/DecodedVector.h" +#include "velox/vector/FlatVector.h" + +namespace facebook::velox::aggregate { +namespace { +int32_t serializedSize(const folly::TDigest& digest) { + return sizeof(size_t) + // maxSize + 4 * sizeof(double) + // sum, count, min, max + sizeof(size_t) + // number of centroids + 2 * sizeof(double) * digest.getCentroids().size(); +} + +template +void serialize(const folly::TDigest& digest, TByteStream& output) { + output.appendOne(digest.maxSize()); + output.appendOne(digest.sum()); + output.appendOne(digest.count()); + output.appendOne(digest.min()); + output.appendOne(digest.max()); + + const auto& centroids = digest.getCentroids(); + output.appendOne(centroids.size()); + for (const auto& centroid : centroids) { + output.appendOne(centroid.mean()); + output.appendOne(centroid.weight()); + } +} + +template +folly::TDigest deserialize(TByteStream& input) { + auto maxSize = input.template read(); + auto sum = input.template read(); + auto count = input.template read(); + auto min = input.template read(); + auto max = input.template read(); + + auto centroidCount = input.template read(); + std::vector centroids; + centroids.reserve(centroidCount); + for (auto i = 0; i < centroidCount; i++) { + auto mean = input.template read(); + auto weight = input.template read(); + centroids.emplace_back(folly::TDigest::Centroid(mean, weight)); + } + + return folly::TDigest(std::move(centroids), sum, count, max, min, maxSize); +} + +template +folly::TDigest singleValueDigest(T v, int64_t count) { + return folly::TDigest( + {folly::TDigest::Centroid(v, count)}, v * count, count, v, v); +} + +struct TDigestAccumulator { + explicit TDigestAccumulator(exec::HashStringAllocator* allocator) + : values_{exec::StlAllocator(allocator)}, + largeCountValues_{exec::StlAllocator(allocator)}, + largeCounts_{exec::StlAllocator(allocator)} {} + + void write( + const folly::TDigest& digest, + exec::HashStringAllocator* allocator) { + if (!begin_) { + begin_ = allocator->allocate(serializedSize(digest)); + } + + ByteStream stream(allocator); + allocator->extendWrite({begin_, begin_->begin()}, stream); + serialize(digest, stream); + allocator->finishWrite(stream, 0); + } + + folly::TDigest read() const { + VELOX_CHECK(begin_); + + ByteStream inStream; + exec::HashStringAllocator::prepareRead(begin_, inStream); + return deserialize(inStream); + } + + bool hasValue() const { + return begin_ != nullptr; + } + + void destroy(exec::HashStringAllocator* allocator) { + if (begin_) { + allocator->free(begin_); + } + } + + template + void append(T v, exec::HashStringAllocator* allocator) { + values_.emplace_back((double)v); + + if (values_.size() >= kMaxBufferSize) { + flush(allocator); + } + } + + template + void append(T v, int64_t count, exec::HashStringAllocator* allocator) { + static const int64_t kMaxCountToBuffer = 99; + + if (values_.size() + count <= kMaxBufferSize && + count <= kMaxCountToBuffer) { + values_.reserve(values_.size() + count); + for (auto i = 0; i < count; i++) { + values_.emplace_back((double)v); + } + + if (values_.size() >= kMaxBufferSize) { + flush(allocator); + } + } else { + largeCountValues_.emplace_back(v); + largeCounts_.emplace_back(count); + if (largeCountValues_.size() >= kMaxBufferSize) { + flush(allocator); + } + } + } + + void append(folly::TDigest digest, exec::HashStringAllocator* allocator) { + if (hasValue()) { + auto currentDigest = read(); + std::vector digests = { + std::move(currentDigest), std::move(digest)}; + auto combinedDigest = + folly::TDigest::merge(folly::Range(digests.data(), digests.size())); + write(combinedDigest, allocator); + } else { + write(digest, allocator); + } + } + + void flush(exec::HashStringAllocator* allocator) { + if (!values_.empty()) { + folly::TDigest digest{hasValue() ? read() : folly::TDigest()}; + digest = digest.merge(values_); + values_.clear(); + write(digest, allocator); + } + + if (!largeCountValues_.empty()) { + std::vector digests; + digests.reserve(largeCountValues_.size() + 1); + for (auto i = 0; i < largeCountValues_.size(); i++) { + digests.emplace_back( + singleValueDigest(largeCountValues_[i], largeCounts_[i])); + } + + if (hasValue()) { + digests.emplace_back(read()); + } + + auto combinedDigest = + folly::TDigest::merge(folly::Range(digests.data(), digests.size())); + largeCountValues_.clear(); + largeCounts_.clear(); + + write(combinedDigest, allocator); + } + } + + private: + // Maximum number of values to accumulate before updating TDigest. + static const size_t kMaxBufferSize = 4096; + + exec::HashStringAllocator::Header* begin_{nullptr}; + + std::vector> values_; + + std::vector> largeCountValues_; + std::vector> largeCounts_; +}; + +// The following variations are possible: +// x, percentile +// x, weight, percentile +// x, percentile, accuracy (not supported yet) +// x, weight, percentile, accuracy (not supported yet) +template +class ApproxPercentileAggregate : public exec::Aggregate { + public: + ApproxPercentileAggregate( + core::AggregationNode::Step step, + bool hasWeight, + const TypePtr& resultType) + : exec::Aggregate(step, resultType), hasWeight_{hasWeight} {} + + int32_t accumulatorFixedWidthSize() const override { + return sizeof(TDigestAccumulator); + } + + void initializeNewGroups( + char** groups, + folly::Range indices) override { + exec::Aggregate::setAllNulls(groups, indices); + for (auto i : indices) { + auto group = groups[i]; + new (group + offset_) TDigestAccumulator(allocator_); + } + } + + void destroy(folly::Range groups) override { + for (auto group : groups) { + auto accumulator = value(group); + accumulator->destroy(allocator_); + } + } + + void finalize(char** groups, int32_t numGroups) override { + for (auto i = 0; i < numGroups; ++i) { + auto accumulator = value(groups[i]); + accumulator->flush(allocator_); + } + } + + void extractValues(char** groups, int32_t numGroups, VectorPtr* result) + override { + VELOX_CHECK(result); + auto flatResult = (*result)->asFlatVector(); + + extract( + groups, + numGroups, + flatResult, + [&](const folly::TDigest& digest, + FlatVector* result, + vector_size_t index) { + result->set(index, (T)digest.estimateQuantile(percentile_)); + }); + } + + void extractAccumulators(char** groups, int32_t numGroups, VectorPtr* result) + override { + VELOX_CHECK(result); + auto flatResult = (*result)->asFlatVector(); + + extract( + groups, + numGroups, + flatResult, + [&](const folly::TDigest& digest, + FlatVector* result, + vector_size_t index) { + auto size = sizeof(double) /*percentile*/ + serializedSize(digest); + Buffer* buffer = flatResult->getBufferWithSpace(size); + StringView serialized(buffer->as() + buffer->size(), size); + OutputByteStream stream(buffer->asMutable() + buffer->size()); + stream.appendOne(percentile_); + serialize(digest, stream); + buffer->setSize(buffer->size() + size); + result->setNoCopy(index, serialized); + }); + } + + protected: + void updatePartial( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodeArguments(rows, args); + checkSetPercentile(); + + if (hasWeight_) { + rows.applyToSelected([&](auto row) { + if (decodedValue_.isNullAt(row) || decodedWeight_.isNullAt(row)) { + return; + } + + auto accumulator = value(groups[row]); + auto value = decodedValue_.valueAt(row); + auto weight = decodedWeight_.valueAt(row); + VELOX_USER_CHECK_GE( + weight, + 1, + "The value of the weight parameter must be greater than or equal to 1."); + accumulator->append(value, weight, allocator_); + }); + } else { + if (decodedValue_.mayHaveNulls()) { + rows.applyToSelected([&](auto row) { + if (decodedValue_.isNullAt(row)) { + return; + } + + auto accumulator = value(groups[row]); + accumulator->append(decodedValue_.valueAt(row), allocator_); + }); + } else { + rows.applyToSelected([&](auto row) { + auto accumulator = value(groups[row]); + accumulator->append(decodedValue_.valueAt(row), allocator_); + }); + } + } + } + + void updateFinal( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodedDigest_.decode(*args[0], rows, true); + + rows.applyToSelected([&](auto row) { + if (decodedDigest_.isNullAt(row)) { + return; + } + + auto serialized = decodedDigest_.valueAt(row); + InputByteStream stream(serialized.data()); + auto percentile = stream.read(); + checkSetPercentile(percentile); + auto digest = deserialize(stream); + + auto accumulator = value(groups[row]); + accumulator->append(std::move(digest), allocator_); + }); + } + + void updateSingleGroupPartial( + char* group, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodeArguments(rows, args); + checkSetPercentile(); + + auto accumulator = value(group); + + if (hasWeight_) { + rows.applyToSelected([&](auto row) { + if (decodedValue_.isNullAt(row) || decodedWeight_.isNullAt(row)) { + return; + } + + auto value = decodedValue_.valueAt(row); + auto weight = decodedWeight_.valueAt(row); + VELOX_USER_CHECK_GE( + weight, + 1, + "The value of the weight parameter must be greater than or equal to 1."); + accumulator->append(value, weight, allocator_); + }); + } else { + if (decodedValue_.mayHaveNulls()) { + rows.applyToSelected([&](auto row) { + if (decodedValue_.isNullAt(row)) { + return; + } + + accumulator->append(decodedValue_.valueAt(row), allocator_); + }); + } else { + rows.applyToSelected([&](auto row) { + accumulator->append(decodedValue_.valueAt(row), allocator_); + }); + } + } + } + + void updateSingleGroupFinal( + char* group, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodedDigest_.decode(*args[0], rows, true); + auto accumulator = value(group); + + std::vector digests; + digests.reserve(rows.end() + 1); + if (accumulator->hasValue()) { + digests.emplace_back(accumulator->read()); + } + + rows.applyToSelected([&](auto row) { + if (decodedDigest_.isNullAt(row)) { + return; + } + + auto serialized = decodedDigest_.valueAt(row); + InputByteStream stream(serialized.data()); + auto percentile = stream.read(); + checkSetPercentile(percentile); + + digests.emplace_back(deserialize(stream)); + }); + + if (!digests.empty()) { + auto digest = + folly::TDigest::merge(folly::Range(digests.data(), digests.size())); + accumulator->write(digest, allocator_); + } + } + + private: + template + void extract( + char** groups, + int32_t numGroups, + FlatVector* result, + ExtractFunc extractFunction) { + VELOX_CHECK(result); + result->resize(numGroups); + + uint64_t* rawNulls = nullptr; + if (result->mayHaveNulls()) { + BufferPtr nulls = result->mutableNulls(result->size()); + rawNulls = nulls->asMutable(); + } + + for (auto i = 0; i < numGroups; ++i) { + char* group = groups[i]; + auto accumulator = value(group); + if (!accumulator->hasValue()) { + result->setNull(i, true); + } else { + if (rawNulls) { + bits::clearBit(rawNulls, i); + } + auto digest = accumulator->read(); + extractFunction(digest, result, i); + } + } + } + + void decodeArguments( + const SelectivityVector& rows, + const std::vector& args) { + size_t argIndex = 0; + decodedValue_.decode(*args[argIndex++], rows, true); + if (hasWeight_) { + decodedWeight_.decode(*args[argIndex++], rows, true); + } + decodedPercentile_.decode(*args[argIndex++], rows, true); + + // TODO Add support for accuracy parameter + VELOX_CHECK_EQ(argIndex, args.size()); + } + + void checkSetPercentile() { + VELOX_CHECK( + decodedPercentile_.isConstantMapping(), + "Percentile argument must be constant for all input rows"); + + auto percentile = decodedPercentile_.valueAt(0); + checkSetPercentile(percentile); + } + + void checkSetPercentile(double percentile) { + VELOX_USER_CHECK_GE(percentile, 0, "Percentile must be between 0 and 1"); + VELOX_USER_CHECK_LE(percentile, 1, "Percentile must be between 0 and 1"); + + if (percentile_ < 0) { + percentile_ = percentile; + } else { + VELOX_USER_CHECK_EQ( + percentile, + percentile_, + "Percentile argument must be constant for all input rows"); + } + } + + const bool hasWeight_; + double percentile_{-1.0}; + DecodedVector decodedValue_; + DecodedVector decodedWeight_; + DecodedVector decodedPercentile_; + DecodedVector decodedDigest_; +}; + +bool registerApproxPercentile(const std::string& name) { + exec::AggregateFunctions().Register( + name, + [name]( + core::AggregationNode::Step step, + const std::vector& argTypes, + const TypePtr& resultType) -> std::unique_ptr { + auto isRawInput = exec::isRawInput(step); + auto isPartialOutput = exec::isPartialOutput(step); + auto hasWeight = argTypes.size() == 3; + + TypePtr type = isRawInput ? argTypes[0] : resultType; + + if (isRawInput) { + VELOX_USER_CHECK_GE( + argTypes.size(), 2, "{} takes 2 or 3 arguments", name); + VELOX_USER_CHECK_LE( + argTypes.size(), 3, "{} takes 2 or 3 arguments", name); + + if (hasWeight) { + VELOX_USER_CHECK_EQ( + argTypes[1]->kind(), + TypeKind::BIGINT, + "The type of the weight argument of {} must be BIGINT", + name); + } + + VELOX_USER_CHECK_EQ( + argTypes.back()->kind(), + TypeKind::DOUBLE, + "The type of the percentile argument of {} must be DOUBLE", + name); + } else { + VELOX_USER_CHECK_EQ( + argTypes.size(), + 1, + "The type of partial result for {} must be VARBINARY", + name); + VELOX_USER_CHECK_GE( + argTypes[0]->kind(), + TypeKind::VARBINARY, + "The type of partial result for {} must be VARBINARY", + name); + } + + if (step == core::AggregationNode::Step::kIntermediate) { + return std::make_unique>( + step, false, VARBINARY()); + } + + auto aggResultType = + isPartialOutput ? VARBINARY() : (isRawInput ? type : resultType); + + switch (type->kind()) { + case TypeKind::TINYINT: + return std::make_unique>( + step, hasWeight, aggResultType); + case TypeKind::SMALLINT: + return std::make_unique>( + step, hasWeight, aggResultType); + case TypeKind::INTEGER: + return std::make_unique>( + step, hasWeight, aggResultType); + case TypeKind::BIGINT: + return std::make_unique>( + step, hasWeight, aggResultType); + case TypeKind::REAL: + return std::make_unique>( + step, hasWeight, aggResultType); + case TypeKind::DOUBLE: + return std::make_unique>( + step, hasWeight, aggResultType); + default: + VELOX_USER_FAIL( + "Unsupported input type for {} aggregation {}", + name, + type->toString()); + } + }); + return true; +} + +static bool FB_ANONYMOUS_VARIABLE(g_AggregateFunction) = + registerApproxPercentile(kApproxPercentile); + +} // namespace +} // namespace facebook::velox::aggregate diff --git a/velox/aggregates/Arbitrary.cpp b/velox/aggregates/Arbitrary.cpp new file mode 100644 index 000000000000..137d7c5c2a40 --- /dev/null +++ b/velox/aggregates/Arbitrary.cpp @@ -0,0 +1,287 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "velox/aggregates/AggregateNames.h" +#include "velox/aggregates/SimpleNumerics.h" +#include "velox/aggregates/SingleValueAccumulator.h" +#include "velox/exec/Aggregate.h" +#include "velox/vector/DecodedVector.h" +#include "velox/vector/FlatVector.h" + +namespace facebook::velox::aggregate { + +namespace { + +// Arbitrary aggregate returns any arbitrary non-NULL value. +// We always keep the first (non-NULL) element seen. +template +class Arbitrary : public SimpleNumericAggregate { + public: + explicit Arbitrary(core::AggregationNode::Step step, TypePtr resultType) + : SimpleNumericAggregate(step, resultType) {} + + int32_t accumulatorFixedWidthSize() const override { + return sizeof(T); + } + + void initializeNewGroups( + char** groups, + folly::Range indices) override { + exec::Aggregate::setAllNulls(groups, indices); + } + + void updatePartial( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*unused*/) override { + DecodedVector decoded(*args[0], rows); + + if (decoded.isConstantMapping()) { + if (decoded.isNullAt(0)) { + return; + } + auto value = decoded.valueAt(0); + rows.applyToSelected([&](vector_size_t i) { + if (exec::Aggregate::isNull(groups[i])) { + updateValue(groups[i], value); + } + }); + } else if (decoded.mayHaveNulls()) { + rows.applyToSelected([&](vector_size_t i) { + if (!decoded.isNullAt(i) && exec::Aggregate::isNull(groups[i])) { + updateValue(groups[i], decoded.valueAt(i)); + } + }); + } else { + rows.applyToSelected([&](vector_size_t i) { + if (exec::Aggregate::isNull(groups[i])) { + updateValue(groups[i], decoded.valueAt(i)); + } + }); + } + } + + void updateFinal( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool mayPushdown) override { + updatePartial(groups, rows, args, mayPushdown); + } + + void updateSingleGroupPartial( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool /*unused*/) override { + DecodedVector decoded(*args[0], allRows); + + if (decoded.isConstantMapping()) { + if (decoded.isNullAt(0)) { + return; + } + updateValue(group, decoded.valueAt(0)); + } else if (!decoded.mayHaveNulls()) { + updateValue(group, decoded.valueAt(0)); + } else { + for (vector_size_t i = 0; i < allRows.end(); ++i) { + // Find the first non-null value. + if (!decoded.isNullAt(i)) { + updateValue(group, decoded.valueAt(i)); + return; + } + } + } + } + + void updateSingleGroupFinal( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool mayPushdown) override { + updateSingleGroupPartial(group, allRows, args, mayPushdown); + } + + private: + inline void updateValue(char* group, T value) { + exec::Aggregate::clearNull(group); + *exec::Aggregate::value(group) = value; + } +}; + +// Arbitrary for non-numeric types. We always keep the first (non-NULL) element +// seen. Arbitrary (x) will produce partial and final aggregations of type x. +class NonNumericArbitrary : public exec::Aggregate { + public: + explicit NonNumericArbitrary( + core::AggregationNode::Step step, + const TypePtr& resultType) + : exec::Aggregate(step, resultType) {} + + // We use singleValueAccumulator to save the results for each group. This + // struct will allow us to save variable-width value. + int32_t accumulatorFixedWidthSize() const override { + return sizeof(SingleValueAccumulator); + } + + // Initialize each group, we will not use the null flags because + // SingleValueAccumulator has its own flag. + void initializeNewGroups( + char** groups, + folly::Range indices) override { + for (auto i : indices) { + new (groups[i] + offset_) SingleValueAccumulator(); + } + } + + void finalize(char** /* groups */, int32_t /* numGroups */) override {} + + void extractValues(char** groups, int32_t numGroups, VectorPtr* result) + override { + VELOX_CHECK(result); + (*result)->resize(numGroups); + + auto* rawNulls = exec::Aggregate::getRawNulls(result->get()); + + for (int32_t i = 0; i < numGroups; ++i) { + char* group = groups[i]; + auto accumulator = value(group); + if (!accumulator->hasValue()) { + (*result)->setNull(i, true); + } else { + exec::Aggregate::clearNull(rawNulls, i); + accumulator->read(*result, i); + } + } + } + + void extractAccumulators(char** groups, int32_t numGroups, VectorPtr* result) + override { + extractValues(groups, numGroups, result); + } + + void destroy(folly::Range groups) override { + for (auto group : groups) { + value(group)->destroy(allocator_); + } + } + + void updatePartial( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*unused*/) override { + DecodedVector decoded(*args[0], rows, true); + if (decoded.isConstantMapping() && decoded.isNullAt(0)) { + // nothing to do; all values are nulls + return; + } + + const auto* indices = decoded.indices(); + const auto* baseVector = decoded.base(); + rows.applyToSelected([&](vector_size_t i) { + if (decoded.isNullAt(i)) { + return; + } + auto* accumulator = value(groups[i]); + if (!accumulator->hasValue()) { + accumulator->write(baseVector, indices[i], allocator_); + } + }); + } + + void updateFinal( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool mayPushdown) override { + updatePartial(groups, rows, args, mayPushdown); + } + + void updateSingleGroupPartial( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool /*unused*/) override { + DecodedVector decoded(*args[0], allRows, true); + if (decoded.isConstantMapping() && decoded.isNullAt(0)) { + // nothing to do; all values are nulls + return; + } + + const auto* indices = decoded.indices(); + const auto* baseVector = decoded.base(); + auto* accumulator = value(group); + for (vector_size_t i = 0; i < allRows.end(); ++i) { + // Find the first non-null value. + if (!decoded.isNullAt(i)) { + accumulator->write(baseVector, indices[i], allocator_); + return; + } + } + } + + void updateSingleGroupFinal( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool mayPushdown) override { + updateSingleGroupPartial(group, allRows, args, mayPushdown); + } +}; + +bool registerArbitraryAggregate(const std::string& name) { + exec::AggregateFunctions().Register( + name, + [name]( + core::AggregationNode::Step step, + const std::vector& argTypes, + const TypePtr& + /*resultType*/) -> std::unique_ptr { + VELOX_CHECK_LE(argTypes.size(), 1, "{} takes only one argument", name); + auto inputType = argTypes[0]; + switch (inputType->kind()) { + case TypeKind::TINYINT: + return std::make_unique>(step, inputType); + case TypeKind::SMALLINT: + return std::make_unique>(step, inputType); + case TypeKind::INTEGER: + return std::make_unique>(step, inputType); + case TypeKind::BIGINT: + return std::make_unique>(step, inputType); + case TypeKind::REAL: + return std::make_unique>(step, inputType); + case TypeKind::DOUBLE: + return std::make_unique>(step, inputType); + case TypeKind::VARCHAR: + case TypeKind::ARRAY: + case TypeKind::MAP: + case TypeKind::ROW: + return std::make_unique(step, inputType); + default: + VELOX_FAIL( + "Unknown input type for {} aggregation {}", + name, + inputType->kindName()); + } + }); + return true; +} + +static bool FB_ANONYMOUS_VARIABLE(g_AggregateFunction) = + registerArbitraryAggregate(kArbitrary); + +} // namespace +} // namespace facebook::velox::aggregate diff --git a/velox/aggregates/ArrayAgg.cpp b/velox/aggregates/ArrayAgg.cpp new file mode 100644 index 000000000000..246b4356a388 --- /dev/null +++ b/velox/aggregates/ArrayAgg.cpp @@ -0,0 +1,184 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "velox/aggregates/AggregateNames.h" +#include "velox/aggregates/ValueList.h" +#include "velox/exec/ContainerRowSerde.h" +#include "velox/vector/ComplexVector.h" + +namespace facebook::velox::aggregate { +namespace { + +struct ArrayAccumulator { + ValueList elements; +}; + +class ArrayAggAggregate : public exec::Aggregate { + public: + explicit ArrayAggAggregate( + core::AggregationNode::Step step, + TypePtr resultType) + : Aggregate(step, resultType) {} + + int32_t accumulatorFixedWidthSize() const override { + return sizeof(ArrayAccumulator); + } + + void initializeNewGroups( + char** groups, + folly::Range indices) override { + for (auto index : indices) { + new (groups[index] + offset_) ArrayAccumulator(); + } + } + + void finalize(char** groups, int32_t numGroups) override { + for (auto i = 0; i < numGroups; i++) { + value(groups[i])->elements.finalize(allocator_); + } + } + + void extractValues(char** groups, int32_t numGroups, VectorPtr* result) + override { + auto vector = (*result)->as(); + VELOX_CHECK(vector); + vector->resize(numGroups); + + auto elements = vector->elements(); + elements->resize(countElements(groups, numGroups)); + + uint64_t* rawNulls = getRawNulls(vector); + vector_size_t offset = 0; + for (int32_t i = 0; i < numGroups; ++i) { + clearNull(rawNulls, i); + + auto& values = value(groups[i])->elements; + auto arraySize = values.size(); + ValueListReader reader(values); + for (auto index = 0; index < arraySize; ++index) { + reader.next(*elements, offset + index); + } + vector->setOffsetAndSize(i, offset, arraySize); + offset += arraySize; + } + } + + void extractAccumulators(char** groups, int32_t numGroups, VectorPtr* result) + override { + extractValues(groups, numGroups, result); + } + + void updatePartial( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodedElements_.decode(*args[0], rows); + rows.applyToSelected([&](vector_size_t row) { + auto group = groups[row]; + value(group)->elements.appendValue( + decodedElements_, row, allocator_); + }); + } + + void updateFinal( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + VELOX_CHECK_EQ(args[0]->encoding(), VectorEncoding::Simple::ARRAY); + auto arrayVector = args[0]->as(); + auto& elements = arrayVector->elements(); + rows.applyToSelected([&](vector_size_t row) { + auto group = groups[row]; + value(group)->elements.appendRange( + elements, + arrayVector->offsetAt(row), + arrayVector->sizeAt(row), + allocator_); + }); + } + + void updateSingleGroupPartial( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool /* mayPushdown */) override { + auto& values = value(group)->elements; + + decodedElements_.decode(*args[0], allRows); + allRows.applyToSelected([&](vector_size_t row) { + values.appendValue(decodedElements_, row, allocator_); + }); + } + + void updateSingleGroupFinal( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool /* mayPushdown */) override { + auto& values = value(group)->elements; + + VELOX_CHECK_EQ(args[0]->encoding(), VectorEncoding::Simple::ARRAY); + auto arrayVector = args[0]->as(); + auto elements = arrayVector->elements(); + allRows.applyToSelected([&](vector_size_t row) { + values.appendRange( + elements, + arrayVector->offsetAt(row), + arrayVector->sizeAt(row), + allocator_); + }); + } + + void destroy(folly::Range groups) override { + for (auto group : groups) { + value(group)->elements.free(allocator_); + } + } + + private: + vector_size_t countElements(char** groups, int32_t numGroups) const { + vector_size_t size = 0; + for (int32_t i = 0; i < numGroups; ++i) { + size += value(groups[i])->elements.size(); + } + return size; + } + + // Reusable instance of DecodedVector for decoding input vectors. + DecodedVector decodedElements_; +}; + +bool registerArrayAggregate(const std::string& name) { + exec::AggregateFunctions().Register( + name, + [name]( + core::AggregationNode::Step step, + const std::vector& argTypes, + const TypePtr& + /*resultType*/) -> std::unique_ptr { + VELOX_CHECK_EQ( + argTypes.size(), 1, "{} takes at most one argument", name); + auto rawInput = exec::isRawInput(step); + TypePtr returnType = rawInput ? ARRAY(argTypes[0]) : argTypes[0]; + return std::make_unique(step, returnType); + }); + return true; +} + +static bool FB_ANONYMOUS_VARIABLE(g_AggregateFunction) = + registerArrayAggregate(kArrayAgg); + +} // namespace +} // namespace facebook::velox::aggregate diff --git a/velox/aggregates/AverageAggregates.cpp b/velox/aggregates/AverageAggregates.cpp new file mode 100644 index 000000000000..74072a5a4862 --- /dev/null +++ b/velox/aggregates/AverageAggregates.cpp @@ -0,0 +1,323 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "velox/aggregates/AggregateNames.h" +#include "velox/exec/Aggregate.h" +#include "velox/vector/ComplexVector.h" +#include "velox/vector/DecodedVector.h" +#include "velox/vector/FlatVector.h" + +namespace facebook::velox::aggregate { + +namespace { + +struct SumCount { + double sum{0}; + int64_t count{0}; +}; + +// Partial aggregation produces a pair of sum and count. +// Final aggregation takes a pair of sum and count and returns a double. +// T is the input type for partial aggregation. Not used for final aggregation. +template +class AverageAggregate : public exec::Aggregate { + public: + AverageAggregate(core::AggregationNode::Step step, TypePtr resultType) + : exec::Aggregate(step, resultType) {} + + int32_t accumulatorFixedWidthSize() const override { + return sizeof(SumCount); + } + + void initializeNewGroups( + char** groups, + folly::Range indices) override { + setAllNulls(groups, indices); + for (auto i : indices) { + new (groups[i] + offset_) SumCount(); + } + } + + void finalize(char** /* unused */, int32_t /* unused */) override {} + + void extractValues(char** groups, int32_t numGroups, VectorPtr* result) + override { + auto vector = (*result)->as>(); + VELOX_CHECK(vector); + vector->resize(numGroups); + uint64_t* rawNulls = getRawNulls(vector); + + double* rawValues = vector->mutableRawValues(); + for (int32_t i = 0; i < numGroups; ++i) { + char* group = groups[i]; + if (isNull(group)) { + vector->setNull(i, true); + } else { + clearNull(rawNulls, i); + auto* sumCount = accumulator(group); + rawValues[i] = (double)sumCount->sum / sumCount->count; + } + } + } + + void extractAccumulators(char** groups, int32_t numGroups, VectorPtr* result) + override { + auto rowVector = (*result)->as(); + auto sumVector = rowVector->childAt(0)->asFlatVector(); + auto countVector = rowVector->childAt(1)->asFlatVector(); + + rowVector->resize(numGroups); + uint64_t* rawNulls = getRawNulls(rowVector); + + int64_t* rawCounts = countVector->mutableRawValues(); + double* rawSums = sumVector->mutableRawValues(); + for (auto i = 0; i < numGroups; ++i) { + char* group = groups[i]; + if (isNull(group)) { + rowVector->setNull(i, true); + } else { + clearNull(rawNulls, i); + auto* sumCount = accumulator(group); + rawCounts[i] = sumCount->count; + rawSums[i] = sumCount->sum; + } + } + } + + protected: + void updatePartial( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodedRaw_.decode(*args[0], rows); + if (decodedRaw_.isConstantMapping()) { + if (!decodedRaw_.isNullAt(0)) { + auto value = decodedRaw_.valueAt(0); + rows.applyToSelected( + [&](vector_size_t i) { updateNonNullValue(groups[i], value); }); + } + } else if (decodedRaw_.mayHaveNulls()) { + rows.applyToSelected([&](vector_size_t i) { + if (decodedRaw_.isNullAt(i)) { + return; + } + updateNonNullValue(groups[i], decodedRaw_.valueAt(i)); + }); + } else if (!exec::Aggregate::numNulls_ && decodedRaw_.isIdentityMapping()) { + auto data = decodedRaw_.data(); + rows.applyToSelected([&](vector_size_t i) { + updateNonNullValue(groups[i], data[i]); + }); + } else { + rows.applyToSelected([&](vector_size_t i) { + updateNonNullValue(groups[i], decodedRaw_.valueAt(i)); + }); + } + } + + void updateSingleGroupPartial( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool /*mayPushdown*/) override { + decodedRaw_.decode(*args[0], allRows); + if (decodedRaw_.isConstantMapping()) { + if (!decodedRaw_.isNullAt(0)) { + auto totalSum = decodedRaw_.valueAt(0) * allRows.end(); + updateNonNullValue(group, allRows.end(), totalSum); + } + } else if (decodedRaw_.mayHaveNulls()) { + for (vector_size_t i = 0; i < allRows.end(); i++) { + if (!decodedRaw_.isNullAt(i)) { + updateNonNullValue(group, decodedRaw_.valueAt(i)); + } + } + } else { + double totalSum = 0; + for (vector_size_t i = 0; i < allRows.end(); i++) { + totalSum += decodedRaw_.valueAt(i); + } + updateNonNullValue(group, allRows.end(), totalSum); + } + } + + void updateFinal( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /* mayPushdown */) override { + decodedPartial_.decode(*args[0], rows); + auto baseRowVector = dynamic_cast(decodedPartial_.base()); + auto baseSumVector = baseRowVector->childAt(0)->as>(); + auto baseCountVector = + baseRowVector->childAt(1)->as>(); + + if (decodedPartial_.isConstantMapping()) { + if (!decodedPartial_.isNullAt(0)) { + auto decodedIndex = decodedPartial_.index(0); + auto count = baseCountVector->valueAt(decodedIndex); + auto sum = baseSumVector->valueAt(decodedIndex); + rows.applyToSelected([&](vector_size_t i) { + updateNonNullValue(groups[i], count, sum); + }); + } + } else if (decodedPartial_.mayHaveNulls()) { + rows.applyToSelected([&](vector_size_t i) { + if (decodedPartial_.isNullAt(i)) { + return; + } + auto decodedIndex = decodedPartial_.index(i); + updateNonNullValue( + groups[i], + baseCountVector->valueAt(decodedIndex), + baseSumVector->valueAt(decodedIndex)); + }); + } else { + rows.applyToSelected([&](vector_size_t i) { + auto decodedIndex = decodedPartial_.index(i); + updateNonNullValue( + groups[i], + baseCountVector->valueAt(decodedIndex), + baseSumVector->valueAt(decodedIndex)); + }); + } + } + + void updateSingleGroupFinal( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool /* mayPushdown */) override { + decodedPartial_.decode(*args[0], allRows); + auto baseRowVector = dynamic_cast(decodedPartial_.base()); + auto baseSumVector = baseRowVector->childAt(0)->as>(); + auto baseCountVector = + baseRowVector->childAt(1)->as>(); + if (decodedPartial_.isConstantMapping()) { + if (!decodedPartial_.isNullAt(0)) { + auto decodedIndex = decodedPartial_.index(0); + auto totalCount = + baseCountVector->valueAt(decodedIndex) * allRows.end(); + auto totalSum = baseSumVector->valueAt(decodedIndex) * allRows.end(); + updateNonNullValue(group, totalCount, totalSum); + } + } else if (decodedPartial_.mayHaveNulls()) { + for (vector_size_t i = 0; i < allRows.end(); i++) { + if (!decodedPartial_.isNullAt(i)) { + auto decodedIndex = decodedPartial_.index(i); + updateNonNullValue( + group, + baseCountVector->valueAt(decodedIndex), + baseSumVector->valueAt(decodedIndex)); + } + } + } else { + double totalSum = 0; + int64_t totalCount = 0; + for (vector_size_t i = 0; i < allRows.end(); i++) { + auto decodedIndex = decodedPartial_.index(i); + totalCount += baseCountVector->valueAt(decodedIndex); + totalSum += baseSumVector->valueAt(decodedIndex); + } + updateNonNullValue(group, totalCount, totalSum); + } + } + + private: + // partial + template + inline void updateNonNullValue(char* group, T value) { + if constexpr (tableHasNulls) { + exec::Aggregate::clearNull(group); + } + accumulator(group)->sum += value; + accumulator(group)->count += 1; + } + + inline void updateNonNullValue(char* group, int64_t count, double sum) { + exec::Aggregate::clearNull(group); + accumulator(group)->sum += sum; + accumulator(group)->count += count; + } + + inline SumCount* accumulator(char* group) { + return exec::Aggregate::value(group); + } + + DecodedVector decodedRaw_; + DecodedVector decodedPartial_; +}; + +void checkSumCountRowType(TypePtr type, const std::string& errorMessage) { + VELOX_CHECK_EQ(type->kind(), TypeKind::ROW, "{}", errorMessage); + VELOX_CHECK_EQ( + type->childAt(0)->kind(), TypeKind::DOUBLE, "{}", errorMessage); + VELOX_CHECK_EQ( + type->childAt(1)->kind(), TypeKind::BIGINT, "{}", errorMessage); +} + +bool registerAverageAggregate(const std::string& name) { + exec::AggregateFunctions().Register( + name, + [name]( + core::AggregationNode::Step step, + const std::vector& argTypes, + const TypePtr& /*resultType*/) -> std::unique_ptr { + VELOX_CHECK_LE( + argTypes.size(), 1, "{} takes at most one argument", name); + auto inputType = argTypes[0]; + TypePtr resultType; + if (exec::isPartialOutput(step)) { + resultType = ROW({"sum", "count"}, {DOUBLE(), BIGINT()}); + } else { + resultType = DOUBLE(); + } + if (exec::isRawInput(step)) { + switch (inputType->kind()) { + case TypeKind::SMALLINT: + return std::make_unique>( + step, resultType); + case TypeKind::INTEGER: + return std::make_unique>( + step, resultType); + case TypeKind::BIGINT: + return std::make_unique>( + step, resultType); + case TypeKind::REAL: + return std::make_unique>( + step, resultType); + case TypeKind::DOUBLE: + return std::make_unique>( + step, resultType); + default: + VELOX_FAIL( + "Unknown input type for {} aggregation {}", + name, + inputType->kindName()); + return nullptr; + } + } else { + checkSumCountRowType( + inputType, + "Input type for final aggregation must be (sum:double, count:bigint) struct"); + return std::make_unique>(step, resultType); + } + }); + return true; +} + +static bool FB_ANONYMOUS_VARIABLE(g_AggregateFunction) = + registerAverageAggregate(kAvg); +} // namespace +} // namespace facebook::velox::aggregate diff --git a/velox/aggregates/BitwiseAggregates.cpp b/velox/aggregates/BitwiseAggregates.cpp new file mode 100644 index 000000000000..b50a4465dd17 --- /dev/null +++ b/velox/aggregates/BitwiseAggregates.cpp @@ -0,0 +1,185 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "velox/aggregates/AggregateNames.h" +#include "velox/aggregates/SimpleNumerics.h" +#include "velox/exec/Aggregate.h" + +namespace facebook::velox::aggregate { + +namespace { + +template +class BitwiseAndOrAggregate : public SimpleNumericAggregate { + public: + BitwiseAndOrAggregate( + core::AggregationNode::Step step, + TypePtr resultType, + T initialValue) + : SimpleNumericAggregate(step, resultType), + initialValue_(initialValue) {} + + int32_t accumulatorFixedWidthSize() const override { + return sizeof(T); + } + + void initializeNewGroups( + char** groups, + folly::Range indices) override { + exec::Aggregate::setAllNulls(groups, indices); + for (auto i : indices) { + *exec::Aggregate::value(groups[i]) = initialValue_; + } + } + + void updateFinal( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool mayPushdown) override { + this->updatePartial(groups, rows, args, mayPushdown); + } + + void updateSingleGroupFinal( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool mayPushdown) override { + this->updateSingleGroupPartial(group, allRows, args, mayPushdown); + } + + protected: + const T initialValue_; +}; + +template +class BitwiseOrAggregate : public BitwiseAndOrAggregate { + public: + explicit BitwiseOrAggregate( + core::AggregationNode::Step step, + TypePtr resultType) + : BitwiseAndOrAggregate( + step, + resultType, + /* initialValue = */ 0) {} + + void updatePartial( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool mayPushdown) override { + SimpleNumericAggregate::template updateGroups( + groups, + rows, + args[0], + [](T& result, T value) { result |= value; }, + mayPushdown); + } + + void updateSingleGroupPartial( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool mayPushdown) override { + SimpleNumericAggregate::updateOneGroup( + group, + allRows, + args[0], + [](T& result, T value) { result |= value; }, + [](T& result, T value, int /* unused */ + ) { result |= value; }, + mayPushdown, + this->initialValue_); + } +}; + +template +class BitwiseAndAggregate : public BitwiseAndOrAggregate { + public: + explicit BitwiseAndAggregate( + core::AggregationNode::Step step, + TypePtr resultType) + : BitwiseAndOrAggregate( + step, + resultType, + /* initialValue = */ -1) {} + + void updatePartial( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool mayPushdown) override { + SimpleNumericAggregate::template updateGroups( + groups, + rows, + args[0], + [](T& result, T value) { result &= value; }, + mayPushdown); + } + + void updateSingleGroupPartial( + char* group, + const SelectivityVector& allRows, + const std::vector& args, + bool mayPushdown) override { + SimpleNumericAggregate::template updateOneGroup( + group, + allRows, + args[0], + [](T& result, T value) { result &= value; }, + [](T& result, T value, int /* unused */ + ) { result &= value; }, + mayPushdown, + this->initialValue_); + } +}; + +template