diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..fc3f979 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,17 @@ +[bumpversion] +current_version = 0.0.1-dev +commit = False +tag = False +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? +serialize = + {major}.{minor}.{patch}-{release} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = prod +first_value = dev +values = + dev + prod + +[bumpversion:file:./src/default/VERSION] diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..2c70198 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,15 @@ +[run] +branch = True +source = + src + +omit = + *__init__* + *__version__* + */thirdparty/** + +[report] +exclude_lines = + pragma: no cover + if __name__ == .__main__. +show_missing = True \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..19b2592 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +**/venv/ +docs/ +tests/ +ci/ +.vscode/ +.idea/ +.pytest_cache/ \ No newline at end of file diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..834a9cc --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +# This is a standard to preconfigure editors +# check: https://editorconfig.org/ +root = true + +# 4 space indentation +[*.py] +charset = utf-8 +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = false +end_of_line = lf diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..2a951b0 --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +ignore= + # line too long - is to be checked by black + E501, + # E203 (spaces around :) + E203, + # and W503 (line break before binary operator) are output as a result of Black formatting + W503 + +dictionaries=en_US,python,technical +docstring-convention=google +spellcheck-targets=comments diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..63e99f0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,200 @@ +name: Continuous Integration + +on: + push: + branches: [main, master] + pull_request: + +jobs: + lints: + name: Run linters + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + checks: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Cache pre-commit + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Install pre-commit + run: pip3 install pre-commit + + - name: Run pre-commit checks + run: pre-commit run --all-files --show-diff-on-failure --color always + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: "fs" + ignore-unfixed: true + exit-code: 0 # change if you want to fail build on vulnerabilities + severity: "CRITICAL,HIGH,MEDIUM" + format: "table" + output: "trivy-scanning-results.txt" + + - name: Format trivy message + run: | + echo "Trivy scanning results." >> trivy.txt + cat trivy-scanning-results.txt >> trivy.txt + + - name: Add trivy report to PR + uses: thollander/actions-comment-pull-request@v2 + continue-on-error: true + if: ${{ github.event_name == 'pull_request' }} + with: + filePath: trivy.txt + reactions: "" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + comment_tag: trivy + + - name: Create venv + run: . ./setup_dev_env.sh + + - name: Check licenses + run: ./check_licenses.sh + + - name: Generate pip freeze + run: | + source venv/bin/activate + pip freeze > requirements-freeze.txt + + - name: Publish Artefacts + uses: actions/upload-artifact@v3 + if: always() + continue-on-error: true + with: + name: results + path: | + requirements-freeze.txt + licenses.txt + trivy-scanning-results.txt + retention-days: 30 + + - name: Publish Test Report + uses: actions/upload-artifact@v3 + if: always() + continue-on-error: true + with: + name: test-report + path: report.xml + retention-days: 10 + + - name: Validate package build + run: | + source venv/bin/activate + python -m pip install -U build + python -m build + + - name: Publish Package + uses: actions/upload-artifact@v3 + continue-on-error: true + if: success() + with: + name: packages + path: dist/** + retention-days: 3 + + tests: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + checks: write + pull-requests: write + contents: write # required for advanced coverage reporting (to keep branch) + strategy: + fail-fast: false # do not stop all jobs if one fails + matrix: + include: + - python-version: "3.11" + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache Dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }}-${{ hashFiles('**/setup.cfg') }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Dependencies + run: pip install -r requirements-dev.txt + + - name: Run Tests With Coverage + run: | + # run with coverage to not execute tests twice + coverage run -m pytest -v -p no:warnings --junitxml=report.xml tests/ + coverage report + coverage xml + + - name: Test Report + uses: mikepenz/action-junit-report@v4 + continue-on-error: true + if: always() + with: + report_paths: 'report.xml' + + - name: Publish Test Report + uses: actions/upload-artifact@v3 + continue-on-error: true + if: always() + with: + name: test-report + path: report.xml + retention-days: 10 + + # simpler version for code coverage reporting + # - name: Produce Coverage report + # uses: 5monkeys/cobertura-action@v13 + # continue-on-error: true + # with: + # path: coverage.xml + # minimum_coverage: 70 + # fail_below_threshold: false + + # more complex version for better coverage reporting + - name: Produce the coverage report + uses: insightsengineering/coverage-action@v2 + continue-on-error: true + with: + # Path to the Cobertura XML report. + path: coverage.xml + # Minimum total coverage, if you want to the + # workflow to enforce it as a standard. + # This has no effect if the `fail` arg is set to `false`. + threshold: 60 + # Fail the workflow if the minimum code coverage + # reuqirements are not satisfied. + fail: false + # Publish the rendered output as a PR comment + publish: true + # Create a coverage diff report. + diff: true + # Branch to diff against. + # Compare the current coverage to the coverage + # determined on this branch. + diff-branch: ${{ github.event.repository.default_branch }} + # make report togglable + togglable-report: true + # This is where the coverage reports for the + # `diff-branch` are stored. + # Branch is created if it doesn't already exist'. + diff-storage: _xml_coverage_reports + # A custom title that can be added to the code + # coverage summary in the PR comment. + coverage-summary-title: "Code Coverage Summary" diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..298ce31 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,36 @@ +name: Build documentation + +on: + push: + branches: [main, master] + +jobs: + pages: + runs-on: ubuntu-latest + container: python:3.11 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Cache Dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + # for best results, it is better to generate + # documentation within development environment + - name: Create venv + run: . ./setup_dev_env.sh + + - name: Build docs + run: ./build_docs.sh + + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./public diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d6813d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,84 @@ +# Directories +.vscode/ +.idea/ +.neptune/ +.pytest_cache/ +.mypy_cache/ +venv/ +__pycache__/ +**.egg-info/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Sphinx documentation +docs/_build/ +public/ +# autogenerated package license table +docs/licenses_table.rst + +# license dump file +licenses.txt + +# File formats +*.onnx +*.pyc +*.pt +*.pth +*.pkl +*.mar +*.torchscript +**/.ipynb_checkpoints +**/dist/ +**/checkpoints/ +**/outputs/ + +# Other env files +.python-version +pyvenv.cfg +pip-selfcheck.json + + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# dotenv +.env + +# coverage and pytest reports +coverage.xml +report.xml + +# CMake +cmake-build-*/ \ No newline at end of file diff --git a/.libraries-whitelist.txt b/.libraries-whitelist.txt new file mode 100644 index 0000000..60b6186 --- /dev/null +++ b/.libraries-whitelist.txt @@ -0,0 +1 @@ +pkg_resources \ No newline at end of file diff --git a/.license-whitelist.txt b/.license-whitelist.txt new file mode 100644 index 0000000..b38cf54 --- /dev/null +++ b/.license-whitelist.txt @@ -0,0 +1,24 @@ +3-Clause BSD License +Apache 2 +Apache License 2 +Apache Software License +Apache Software License, BSD License +Apache Software License, MIT License +Apache-2 +Apache License, Version 2 +BSD +BSD License +BSD License, Apache Software License +CC0 1.0 Universal (CC0 1.0) Public Domain Dedication +Freely Distributable +ISC License (ISCL) +MIT +MIT License +MIT License, Mozilla Public License 2.0 (MPL 2.0) +Mozilla Public License 2.0 (MPL 2.0) +Public Domain +Python Software Foundation License +Python Software Foundation License, MIT License +Unlicense +Proprietary License +Historical Permission Notice and Disclaimer (HPND) \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e8a0c69 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,93 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-case-conflict + - id: check-merge-conflict + - id: trailing-whitespace + exclude: .bumpversion.cfg|notebooks/.*\.py + - id: check-ast + - id: check-added-large-files + - id: check-toml + - id: check-json + - id: check-yaml + + # PEP 8 compliant opinionated formatter. + - repo: https://github.com/psf/black + rev: 23.10.1 + hooks: + - id: black + exclude: (docs/|notebooks/) + - id: black-jupyter + files: \.ipynb$ + + # Cleaning unused imports. + - repo: https://github.com/hadialqattan/pycln + rev: v2.3.0 + hooks: + - id: pycln + args: ["-a"] + exclude: (docs/|notebooks/) + + # Modernizes python code and upgrade syntax for newer versions of the language + - repo: https://github.com/asottile/pyupgrade + rev: v3.15.0 + hooks: + - id: pyupgrade + args: [--py38-plus] + + # Used to have proper type annotations for library code. + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.6.1 + hooks: + - id: mypy + args: [] + # You can add additional plugins for mypy below + # such as types-python-dateutil + additional_dependencies: [] + exclude: (/test_|setup.py|/tests/|docs/) + + # Sort imports alphabetically, and automatically separated into sections and by type. + - repo: https://github.com/timothycrosley/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--profile", "black"] + exclude: (docs/|notebooks/) + + # Checks Python source files for errors. + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + name: flake8 + entry: flake8 + language: python + types: [python] + args: [--config, .flake8] + exclude: (docs/) + + # Enforces a coding standard, looks for code smells, and can make suggestions about how the code could be refactored. + - repo: https://github.com/pycqa/pylint + rev: v3.0.1 + hooks: + - id: pylint + exclude: (/test_|tests/|docs/) + # # You can add additional plugins for pylint here, + # here is an example for pydantic, remember to enable it in pyproject.toml + # additional_dependencies: + # - 'pylint_pydantic' + # args: + # # pylint can have issue with python libraries based on C + # # if it fails to find some objects likely you need to add them + # # here: + # ["--extension-pkg-whitelist=pydantic"] + + # Finds common security issues in Python code. + - repo: https://github.com/PyCQA/bandit + rev: 1.7.5 + hooks: + - id: bandit + args: [-c, pyproject.toml, --recursive, src] + additional_dependencies: [".[toml]"] # required for pyproject.toml support + exclude: (notebooks/) diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..23e48d2 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1 @@ +Proprietary License. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a3cea2e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-include src/ py.typed *.pyi VERSION +global-exclude __pycache__ +global-exclude *.py[cod] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..3f3471d --- /dev/null +++ b/README.md @@ -0,0 +1,96 @@ +# default + +Repository is created with deepsense.ai project template boilerplate. Adapt to your needs. +Documentation is available at [https://deepsense-ai.github.io/ds-template/](https://deepsense-ai.github.io/ds-template/). + + +# Setup developer environment + +To start, you need to setup your local machine. + +## Setup venv + +You need to setup virtual environment, simplest way is to run from project root directory: + +```bash +$ . ./setup_dev_env.sh +$ source venv/bin/activate +``` +This will create a new venv and run `pip install -r requirements-dev.txt`. +Last line shows how to activate the environment. + +## Install pre-commit + +To ensure code quality we use pre-commit hook with several checks. Setup it by: + +``` +pre-commit install +``` + +All updated files will be reformatted and linted before the commit. + +To reformat and lint all files in the project, use: + +`pre-commit run --all-files` + +The used linters are configured in `.pre-commit-config.yaml`. You can use `pre-commit autoupdate` to bump tools to the latest versions. + +## Autoreload within notebooks + +When you install project's package add below code (before imports) in your notebook: +``` +# Load the "autoreload" extension +%load_ext autoreload +# Change mode to always reload modules: you change code in src, it gets loaded +%autoreload 2 +``` +Read more about different modes in [documentation](https://ipython.org/ipython-doc/3/config/extensions/autoreload.html). + +All code should be in `src/` to make reusability and review straightforward, keep notebooks simple for exploratory data analysis. +See also [Cookiecutter Data Science opinion](https://drivendata.github.io/cookiecutter-data-science/#notebooks-are-for-exploration-and-communication). + +# Project documentation + +In `docs/` directory are Sphinx RST/Markdown files. + +To build documentation locally, in your configured environment, you can use `build_docs.sh` script: + +```bash +$ ./build_docs.sh +``` + +Then open `public/index.html` file. + +Please read the official [Sphinx documentation](https://www.sphinx-doc.org/en/master/) for more details. + + + +### Github Actions Documentation + +By default **Github Actions** pipelines have `documentation` workflow which will build sphinx documentation automatically on main branch - and it will push it to a branch - it can be hosted on **Github Pages** if you enable it. + +To access it, you need to enable it, on **Github repository -> Settings -> Pages** page select **Deploy from a branch** and select **gh-pages**. Link will appear here after deployment. + +**WARNING:** Only on Github Enterprise you can make it private so only people with repository access can view it. + +Please read more about it [here](https://docs.github.com/en/pages/quickstart).# Semantic version bump + +To bump version of the library please use `bump2version` which will update all version strings. + +NOTE: Configuration is in `.bumpversion.cfg` and **this is a main file defining version which should be updated only with bump2version**. + +For convenience there is bash script which will create commit, to use it call: + +```bash +# to create a new commit by increasing one semvar: +$ ./bump_version.sh minor +$ ./bump_version.sh major +$ ./bump_version.sh patch +# to see what is going to change run: +$ ./bump_version.sh --dry-run major +``` +Script updates **VERSION** file and setup.cfg automatically uses that version. + +You can configure it to update version string in other files as well - please check out the bump2version configuration file. + + diff --git a/build_docs.sh b/build_docs.sh new file mode 100755 index 0000000..091c6c5 --- /dev/null +++ b/build_docs.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Force Sphinx to rebuild documentation - otherwise it generates incosistencies. +rm -rf public/ docs/_build + +# Exit on error for the next commands +set -e -x + +# Activate venv for licenses and also it simplifies sphinx code documentation generation. +. venv/bin/activate + +# Generate a table with all installed libraries, licenses etc +pip-licenses --from=mixed --format rst --with-urls --with-description --output-file=docs/licenses_table.rst + +# Build sphinx docs to public/ directory +sphinx-build -d docs/_build/doctrees docs/ public/ diff --git a/bump_version.sh b/bump_version.sh new file mode 100755 index 0000000..d527d5f --- /dev/null +++ b/bump_version.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +bump2version --verbose --commit $@ \ No newline at end of file diff --git a/check_licenses.sh b/check_licenses.sh new file mode 100755 index 0000000..cd213c7 --- /dev/null +++ b/check_licenses.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +. venv/bin/activate + +pip-licenses --from=mixed --ignore-packages `cat .libraries-whitelist.txt`> licenses.txt +cat licenses.txt + +FOUND=$(tail -n +2 licenses.txt | grep -v -f .license-whitelist.txt | wc -l) + +if [[ $FOUND -gt 0 ]]; then + echo "Detected license/s not on the whitelist ($FOUND found)." + tail -n +2 licenses.txt | grep -v -f .license-whitelist.txt + exit 1 +else + echo "Okay." + exit 0 +fi diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/_templates/.gitkeep b/docs/_templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/api/default.rst b/docs/api/default.rst new file mode 100644 index 0000000..31a6fc8 --- /dev/null +++ b/docs/api/default.rst @@ -0,0 +1,21 @@ +default code documentation +============================================================== + +Submodules +---------- + +default.hello module +---------------------------------------------------- + +.. automodule:: default.hello + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: default + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/modules.rst b/docs/api/modules.rst new file mode 100644 index 0000000..bd696de --- /dev/null +++ b/docs/api/modules.rst @@ -0,0 +1,7 @@ +default modules +======================================================== + +.. toctree:: + :maxdepth: 4 + + default diff --git a/docs/code_documentation.md b/docs/code_documentation.md new file mode 100644 index 0000000..db8467d --- /dev/null +++ b/docs/code_documentation.md @@ -0,0 +1,21 @@ +# Code documentation + +```{hint} + + To add your code use sphinx tool in project root directory: + + $ sphinx-apidoc -o docs/api/ src/default + + and add reference from any page which is reachable from the index page. +``` + +```python + import default +``` + +```{toctree} +--- +maxdepth: 4 +--- +api/modules +``` \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..7f97c47 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,75 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'default' +copyright = '2023, ds' +author = 'deepsense.ai' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.napoleon",# support for google style docstrings + "sphinx.ext.autodoc", # auto* are for automatic code docs generation + "sphinx.ext.autosummary", # as above + "sphinx.ext.intersphinx", # allows to cross reference other sphinx documentations + "sphinx.ext.autosectionlabel", # each doc section gets automatic reference generated + "myst_parser", # adds support for Markdown + "sphinxcontrib.mermaid", # allows to use Mermaid diagrams +] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] + +autosectionlabel_prefix_document = True + +# Mapping to link other documentations +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + "scipy": ("https://docs.scipy.org/doc/scipy/reference/", None), + "numpy": ("https://numpy.org/doc/stable", None), +} + + +# Configuration for Napoleon extension +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_init_with_doc = False +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_preprocess_types = False +napoleon_type_aliases = None +napoleon_attr_annotations = True + +add_module_names = False +autodoc_typehints = 'both' + +source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'markdown', + '.md': 'markdown', +} + +# workaround for sphinx material issue with empty left sidebar +# See: https://github.com/bashtage/sphinx-material/issues/30 +# uncomment below lines if you use: html_theme = "sphinx_material" +# html_sidebars = { +# "**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"] +# } diff --git a/docs/howtos.rst b/docs/howtos.rst new file mode 100644 index 0000000..3e6c438 --- /dev/null +++ b/docs/howtos.rst @@ -0,0 +1,10 @@ +How to..? +============ + +Here is a list of step by step guides for the project. + +.. toctree:: + :maxdepth: 1 + :caption: How to: + + howtos/setup_dev_env diff --git a/docs/howtos/setup_dev_env.rst b/docs/howtos/setup_dev_env.rst new file mode 100644 index 0000000..0265c91 --- /dev/null +++ b/docs/howtos/setup_dev_env.rst @@ -0,0 +1,45 @@ +.. _venv_setup: + +Setup developer environment +============================ + +Setup Python virtual environment +++++++++++++++++++++++++++++++++++ + +There are a few ways to create virtual environment. To boostrap project it is the best to run: + + .. code:: bash + + $ . ./setup_dev_env.sh + +or + + .. code:: bash + + $ source ./setup_dev_env.sh + +This will create a single **venv** directory and install project-related packages. + +.. warning:: This script works only in project root directory. + +To activate the environment in your terminal just type: + + .. code:: bash + + $ source venv/bin/activate + + +Finalize environment +----------------------- +After completing above steps, you will have a virtual Python environment set up and almost ready to use. + +.. warning:: Ensure you installed correct version of pytorch, torchvision and cudatoolkit for **your hardware configuration**. Refer to the official Pytorch guide. + +To install project dependencies run: + +.. code:: bash + + $ pip install -r requirements-dev.txt + +It installs current package in edit mode, developer tools and so on. + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..cea5832 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,27 @@ +.. Documentation master file + You can adapt this file completely to your preferences, but it should at least + contain the root `toctree` directive. + +ds - default +==================================================================================================================== + +Documentation for `default` python package. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + project_overview + code_documentation + howtos + licenses + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/licenses.rst b/docs/licenses.rst new file mode 100644 index 0000000..bc815bc --- /dev/null +++ b/docs/licenses.rst @@ -0,0 +1,6 @@ +Licenses +===================== + +List of automatically detected licenses of all detected python packages with `pip-licenses`. + +.. include:: licenses_table.rst \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..32bb245 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/project_overview.md b/docs/project_overview.md new file mode 100644 index 0000000..082f4e6 --- /dev/null +++ b/docs/project_overview.md @@ -0,0 +1,14 @@ +# Project overview + +This is introduction to the project. + +Write here what this project is about, what problems does it solves on high level. + +You can also use [mermaid](https://mermaid.js.org/) - read more on [extension page](https://github.com/mgaitan/sphinxcontrib-mermaid): + +```{mermaid} + graph TD; + default --> B; + default --> C; + B --> D; +``` \ No newline at end of file diff --git a/notebooks/example.ipynb b/notebooks/example.ipynb new file mode 100644 index 0000000..6273afc --- /dev/null +++ b/notebooks/example.ipynb @@ -0,0 +1,33 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# IPython extension to reload modules before executing user code.\n", + "# For more info check https://ipython.org/ipython-doc/3/config/extensions/autoreload.html\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import default" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b28aac0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,145 @@ +[build-system] +requires = [ + "setuptools >= 40.9.0", + "wheel", +] +build-backend = "setuptools.build_meta" + +[tool.isort] +multi_line_output=3 +line_length=120 +include_trailing_comma = true +known_first_party=[ + 'default' +] +known_third_party=[ # Most popular libraries. Extend if necessary. + 'IPython', + 'PIL', + 'cv2', + 'dotenv', + 'editdistance', + 'fastapi', + 'fire', + 'hydra', + 'joblib', + 'loguru', + 'luigi', + 'matplotlib', + 'neptune', + 'neptune_config', + 'nltk', + 'numpy', + 'omegaconf', + 'pandas', + 'pqdm', + 'pydantic', + 'pytest', + 'pytorch_lightning', + 'requests', + 'scipy', + 'setuptools', + 'shapely', + 'skimage', + 'sklearn', + 'streamlit', + 'torch', + 'torchvision', + 'tqdm', + 'typer', +] +skip_gitignore=true + +[tool.black] +line_length=120 + +[tool.pytest] +norecursedirs=[ + '.git', + '.tox', + '.env', + 'dist', + 'build', + 'migrations', + 'docker', + 'config', + 'notebooks', + 'research', +] +python_files = ['test_*.py'] +addopts = [ + '-ra', + '--showlocals', + '--strict-markers', + '--ignore=docs/conf.py', + '--ignore=setup.py', + '--ignore=ci', + '--ignore=.eggs', + '--doctest-modules', + '--doctest-glob=\*.rst', + '--tb=short', +] + +testpaths = ['tests'] + +[tool.mypy] +warn_unused_configs = true +ignore_missing_imports = true +warn_unused_ignores = false +show_error_codes = true +check_untyped_defs = true +no_implicit_optional = true +mypy_path=['src'] + +[[tool.mypy.overrides]] +module = "default.*" +ignore_missing_imports = false +disallow_untyped_defs = true + +[tool.pylint.basic] +good-names="i,j,x,y,z,x1,y1,z1,x2,y2,z2,cv,df,dx,dy,dz,w,h,c,b,g,qa,q,a" +max-args=8 + +[tool.pylint.main] +load-plugins=["pylint.extensions.docparams"] + +[tool.pylint.messages_control] +disable=[ + "suppressed-message", + # therefore we wouldn't have to install full dependency set in order to lint + "import-error", + # sometimes we create a dataclass or Pydantic module and just don't need public methods + "too-few-public-methods", + # below is handled by pycln + "unused-import", + # below is handled by isort + "wrong-import-order", + # too restrictive + "too-many-instance-attributes", + # not necessary nor useful in our projects + "missing-module-docstring" + ] + +[tool.pylint.format] +max-line-length=120 + +[tool.pylint.miscellaneous] +notes=["XXX"] + +[tool.pylint.parameter_documentation] +accept-no-param-doc = false +accept-no-raise-doc = false +accept-no-return-doc = false +accept-no-yields-doc = false +default-docstring-type = "google" + +[tool.pylint.design] +max-locals=20 + +[tool.pylint.similarities] +min-similarity-lines=10 + +[tool.bandit] +exclude_dirs = ["venv",] +# B101 disables errors for asserts in the code +# remember to not use asserts for security and control flows +skips = ["B101"] \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..0c0f3b0 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,17 @@ +# Requirements as needed for development for this project. +# --------------------------------------------------------- +# Install current project +-e. +# developer tools: +pre-commit +pytest>=6.2.5 +pytest-cov +# for getting dependency licenses and docs: +pip-licenses>=4.0.0,<5.0.0 +# for building docs +sphinx>=5.3.0,<6.0.0 +sphinx-rtd-theme>=1.1.1,<2.0.0 +myst-parser # adds markdown to sphinx +sphinxcontrib-mermaid # adds option to have diagrams in sphinx +# for bumping version strings +bump2version>=1.0.1,<2.0.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..f119c8c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,45 @@ +[metadata] +name = default +# do not change version by hand: use bump_version.sh +version = file: src/default/VERSION +description = "deepsense.ai project" +author = deepsense.ai +author_email = contact@deepsense.ai +license = Other/Proprietary License +license_files = LICENSE.md +classifiers = + Development Status :: 1 - Planning + Environment :: Console + Environment :: GPU :: NVIDIA CUDA + Intended Audience :: Science/Research + License :: Other/Proprietary License + Natural Language :: English + Operating System :: Independent + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Topic :: AI + Private :: Do Not Upload + +[options] +package_dir= + =src +packages=find: +zip_safe = False +platforms = any +include_package_data = True +python_requires = >=3.8 +install_requires = + python-dotenv>=0.5.1 + +[options.packages.find] +where=src + +[bdist_wheel] +universal = 1 + +[aliases] +# Alias `setup.py test` to `setup.py pytest` +test = pytest diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..5850735 --- /dev/null +++ b/setup.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +"""Packaging code - boilerplate required by pip <= 21.1 for development install (-e).""" +from setuptools import setup + +setup() diff --git a/setup_dev_env.sh b/setup_dev_env.sh new file mode 100755 index 0000000..5290d7a --- /dev/null +++ b/setup_dev_env.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ ! -d venv ]; then + python3.11 -m venv venv + . venv/bin/activate + pip install --upgrade pip + pip install --quiet wheel==0.41.3 + pip install --quiet -r requirements-dev.txt +fi + +. venv/bin/activate diff --git a/src/default/VERSION b/src/default/VERSION new file mode 100644 index 0000000..772a55e --- /dev/null +++ b/src/default/VERSION @@ -0,0 +1 @@ +0.0.1-dev \ No newline at end of file diff --git a/src/default/__init__.py b/src/default/__init__.py new file mode 100644 index 0000000..ed1db17 --- /dev/null +++ b/src/default/__init__.py @@ -0,0 +1,4 @@ +""" default """ +from .__version__ import __version__ + +__all__ = ["__version__"] diff --git a/src/default/__version__.py b/src/default/__version__.py new file mode 100644 index 0000000..4d271e8 --- /dev/null +++ b/src/default/__version__.py @@ -0,0 +1,5 @@ +"""Version information.""" +from pathlib import Path + +version_file = Path(__file__).absolute().parents[0] / "VERSION" +__version__ = version_file.read_text(encoding="utf-8").strip() diff --git a/src/default/hello.py b/src/default/hello.py new file mode 100644 index 0000000..d50886d --- /dev/null +++ b/src/default/hello.py @@ -0,0 +1,6 @@ +"""Remove this file. Required only to run coverage on empty project.""" + + +def hello_world() -> None: + """Says hello world.""" + print("hello world") diff --git a/src/default/py.typed b/src/default/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/py.typed b/src/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_default.py b/tests/test_default.py new file mode 100644 index 0000000..9f2d549 --- /dev/null +++ b/tests/test_default.py @@ -0,0 +1,6 @@ +import default.hello + + +def test_basic_test() -> None: + default.hello.hello_world() + assert True