diff --git a/.gitignore b/.gitignore index 3a5d60dda..7378e222a 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ coverage.xml /.pytest_cache/ /.mypy_cache/ taskcat_outputs/ -docs/apidocs/ # Doc Site site/ *.zip diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a12f0d31d..64729d903 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: ^(.travis.yml|.pre-commit-config.yaml|.bumpversion.cfg)$ +exclude: ^(.pre-commit-config.yaml|.bumpversion.cfg)$ fail_fast: true repos: - repo: https://github.com/pre-commit/mirrors-isort diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c0a4e9d46..000000000 --- a/.travis.yml +++ /dev/null @@ -1,144 +0,0 @@ -# taskcat ci config version v2 -# Python Versions -# Python: 3.7 -# -# Create development builds (Triggered by commit to develop branch) -# - Builds pypi package to https://test.pypi.org/project/taskcat/ -# - Tags development builds -# -dist: xenial -language: python - -branches: - only: - - main - -python: - - "3.7.2" - - "3.8" -before_install: - - pip install -r travis-specific-requirements.txt - - pip install -r dev-requirements.txt - - cd /tmp - - GIT_SECRETS_RELEASE=1.3.0 - - wget https://github.com/awslabs/git-secrets/archive/${GIT_SECRETS_RELEASE}.tar.gz - - tar -xvf ${GIT_SECRETS_RELEASE}.tar.gz - - cd git-secrets-${GIT_SECRETS_RELEASE}/ - - sudo make install - - cd ${TRAVIS_BUILD_DIR} -install: - - pip install . -script: pre-commit run --all-files - -jobs: - include: - - stage: "[Version Bump] Check/Create Release PR" - name: "[Version Bump] Check/Create Release PR" - if: branch = main AND commit_message !~ /(Bump|Release taskcat)/ AND type = push - before_script: - - | - pip install bump2version - HUB_DIST=linux-amd64 - HUB_VERSION=$(curl -w "%{url_effective}\n" -I -L -s -S github.com/github/hub/releases/latest -o /dev/null | awk -F'releases/tag/v' '{ print $2 }') - curl "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-${HUB_DIST}-${HUB_VERSION}.tgz" -L | tar xvz && sudo ./hub-${HUB_DIST}-${HUB_VERSION}/install && rm -r ./hub-${HUB_DIST}-${HUB_VERSION} - - mkdir -p ~/.config/taskcat - - | - echo -e "github.com:\n- user: aws-ia-ci\n oauth_token: ${GHT}\n protocol: https" > ~/.config/hub - script: - - export RELEASE_ID=$(cut -f1-2 -d . VERSION) - - export ORIGINAL_VERSION=$(cat VERSION) - - git checkout "release/v${RELEASE_ID}" || git checkout -b "release/v${RELEASE_ID}" - - bumpversion --no-commit --allow-dirty --no-tag patch - - bumpversion --commit --allow-dirty --no-tag release --message "Bump Version - Creating a new minor version" - - export NEW_VERSION=$(cat VERSION) - - git push "https://${GHT}:@github.com/${TRAVIS_REPO_SLUG}" "release/v${RELEASE_ID}" --force || travis_terminate 1 - - | - OPEN_PR=$(hub pr list -s open --base main --head "release/v${RELEASE_ID}" -L 1 -f "%I") - if [ -z "${OPEN_PR}" ]; then - hub pull-request -m "Release taskcat [${NEW_VERSION}]" -h "release/v${RELEASE_ID}" - fi - - | - OPEN_PR=$(hub pr list -s open --base main --head "release/v${RELEASE_ID}" -L 1 -f "%I") - LAST_RELEASE_COMMIT=$(git rev-list --tags --max-count=1) - TAG_BODY=$(git --no-pager log --no-merges --oneline ${LAST_RELEASE_COMMIT}..HEAD --pretty='- %h %s') - hub api -XPATCH repos/${TRAVIS_REPO_SLUG}/issues/${OPEN_PR} -f body="${TAG_BODY}" - - - stage: "[Version Bump] Create PyPI Development release" - name: "[Version Bump] Create PyPI Development release" - if: branch = main AND type = push - before_script: - - pip install bump2version - - pip install packaging - - export UPSTREAM_PYPI_VERSION=$(python -c "from packaging import version; import requests; versions = requests.get('https://pypi.org/pypi/taskcat/json').json()['releases'].keys(); versions = [version.Version(x) for x in versions]; print(sorted(versions, reverse=True)[0])") - script: - - | - echo "${UPSTREAM_PYPI_VERSION}" | egrep -i '\.dev[0-9]{1,4}' - if [[ $? -eq 0 ]]; then - echo "Bumping the development version" - # Replacing VERSION (ex: 0.9.12) with upstream value (ex: 0.9.13.dev0) - sed -i -e "s,$(cat VERSION),${UPSTREAM_PYPI_VERSION},g" .bumpversion.cfg - sed -i -e "s,$(cat VERSION),${UPSTREAM_PYPI_VERSION},g" VERSION - # Now bumping 0.9.13.dev0 -> 0.9.13.dev1 - bumpversion --allow-dirty --no-tag --no-commit build - export NEW_DEV_BUILD=true - else - # v0.9.0 -> v0.9.1.dev0 - bumpversion --allow-dirty --no-tag --no-commit patch - fi - - cat VERSION - - | - if [[ "$(cat VERSION)" == "${UPSTREAM_PYPI_VERSION}" ]]; then - echo "Something went wrong when bumping the version. Exiting." - travis_terminate 1 - fi - - | - egrep -i '\.dev[0-9]{1,4}' VERSION - if [[ $? -eq 1 ]]; then - echo "No .dev pre-release tag found in VERSION. Not building PYPI package" - travis_terminate 1 - fi - deploy: - - provider: pypi - skip_cleanup: true - user: $PYPI_USER - password: $PYPI_PASSWORD - on: - branch: main - - - stage: "Tag build changelog/Push pypi and github release/Update docs" - name: "Tag build changelog/Push pypi and github release/Update docs" - if: branch = main AND fork = false AND type = push - script: - - | - echo "${TRAVIS_COMMIT_MESSAGE}" | egrep -i 'Merge pull request.*from aws-ia/release.*$' - if [[ $? -eq 0 ]]; then - LAST_RELEASE_COMMIT=$(git rev-list --tags --max-count=1) - TAG_BODY=$(git --no-pager log --no-merges --oneline ${LAST_RELEASE_COMMIT}..HEAD --pretty='- %h %s') - git tag -a "$(cat VERSION)" -m "${TAG_BODY}" - git push --tags "https://$GHT:@github.com/$TRAVIS_REPO_SLUG" - export RELEASE_THE_KRAKEN=true - fi - deploy: - - provider: releases - skip_cleanup: true - api_key: "$GHT" - file: directory/* - on: - branch: main - fork: false - condition: $RELEASE_THE_KRAKEN = true - - - provider: pypi - skip_cleanup: true - user: $PYPI_USER - password: $PYPI_PASSWORD - on: - branch: main - fork: false - condition: $RELEASE_THE_KRAKEN = true - - # Add Docker provider -cache: - directories: - - $HOME/.cache/pip - - $HOME/.cache/pre-commit diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md new file mode 100644 index 000000000..a039de99a --- /dev/null +++ b/DOCUMENTATION.md @@ -0,0 +1,167 @@ +# TaskCat Documentation Generation + +This guide explains how to generate and maintain TaskCat's documentation using the automated `gendocs.sh` script. + +## Quick Start + +```bash +# 1. First-time setup (installs dependencies and creates structure) +./gendocs.sh --install + +# 2. Preview documentation locally +./gendocs.sh --preview + +# 3. Build for production +./gendocs.sh --build + +# 4. Deploy to GitHub Pages +./gendocs.sh --deploy +``` + +## Script Options + +| Option | Short | Description | +|--------|-------|-------------| +| `--install` | `-i` | Install dependencies and create documentation structure | +| `--preview` | `-p` | Start local development server at http://localhost:8000 | +| `--build` | `-b` | Build static documentation site | +| `--deploy` | `-d` | Deploy to GitHub Pages | +| `--clean` | `-c` | Clean build artifacts | +| `--help` | `-h` | Show help message | + +## Documentation Structure + +After running `--install`, the following structure is created: + +``` +docs/ +├── index.md # Home page +├── installation.md # Installation guide +├── quickstart.md # Quick start guide +├── configuration.md # Configuration reference +├── user-guide/ # User guides +│ ├── template-testing.md +│ ├── multi-region.md +│ └── parameter-overrides.md +├── examples/ # Usage examples +│ ├── basic.md +│ └── advanced.md +├── reference/ # Auto-generated API docs +└── assets/ # Images and static files +``` + +## Features + +### Modern Design +- Material Design theme with dark/light mode toggle +- Responsive layout for mobile and desktop +- Professional appearance suitable for enterprise use + +### Auto-Generated API Documentation +- Automatically extracts docstrings from Python code +- Generates comprehensive API reference +- Maintains links between related functions and classes + +### Developer-Friendly +- Live reload during development +- Syntax highlighting for code blocks +- Tabbed content for multiple formats (YAML/JSON) +- Admonitions for tips, warnings, and notes + +### GitHub Integration +- One-command deployment to GitHub Pages +- Automatic edit links to source files +- Social links and repository information + +## Customization + +### Adding Content +1. Create new `.md` files in the `docs/` directory +2. Update the `nav` section in `mkdocs.yml` +3. Use standard Markdown with Material extensions + +### Styling +- Modify `mkdocs.yml` for theme customization +- Add custom CSS in `docs/assets/css/` +- Update colors, fonts, and layout options + +### API Documentation +The API documentation is automatically generated from your Python docstrings. The script: +1. Scans all Python files in the `taskcat/` directory +2. Extracts Google-style docstrings +3. Creates cross-referenced documentation +4. Maintains source code links + +## Deployment + +### GitHub Pages Setup +1. Enable GitHub Pages in repository settings +2. Set source to "GitHub Actions" or "gh-pages branch" +3. Run `./gendocs.sh --deploy` to publish + +### Continuous Integration +Add to your GitHub Actions workflow: + +```yaml +- name: Generate Documentation + run: | + ./gendocs.sh --install + ./gendocs.sh --build + ./gendocs.sh --deploy +``` + +## Troubleshooting + +### Common Issues + +**Python/pip not found:** +```bash +# Install Python 3.8+ and pip +# On macOS: brew install python +# On Ubuntu: apt-get install python3 python3-pip +``` + +**MkDocs command not found:** +```bash +./gendocs.sh --install # Reinstall dependencies +``` + +**Port 8000 already in use:** +```bash +# Kill existing process or use different port +mkdocs serve --dev-addr=localhost:8001 +``` + +### Getting Help +- Run `./gendocs.sh --help` for usage information +- Check MkDocs documentation: https://www.mkdocs.org/ +- Material theme docs: https://squidfunk.github.io/mkdocs-material/ + +## Best Practices + +1. **Keep docstrings updated** - API docs are generated from code +2. **Use consistent formatting** - Follow Google docstring style +3. **Add examples** - Include code examples in documentation +4. **Test locally** - Always preview before deploying +5. **Version control** - Commit documentation changes with code changes + +## Advanced Usage + +### Custom Plugins +Add plugins to `mkdocs.yml`: +```yaml +plugins: + - search + - mkdocstrings + - mermaid2 # For diagrams + - pdf-export # For PDF generation +``` + +### Multiple Versions +For version-specific documentation: +```bash +mike deploy --push --update-aliases 1.0 latest +mike set-default --push latest +``` + +This creates a professional documentation site that automatically stays in sync with your codebase and provides an excellent user experience for TaskCat users and contributors. diff --git a/README.md b/README.md deleted file mode 100644 index 344f3b226..000000000 --- a/README.md +++ /dev/null @@ -1,35 +0,0 @@ -[![logo](https://raw.githubusercontent.com/aws-ia/taskcat/main/assets/docs/images/tcat.png)](https://taskcat.io/) -[![Build Status](https://travis-ci.com/aws-ia/taskcat.svg?branch=main)](https://travis-ci.com/aws-ia/taskcat) [![PyPI version](https://badge.fury.io/py/taskcat.svg)](https://badge.fury.io/py/taskcat) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) - - - -## What is TaskCat? -**TaskCat** is a tool that tests AWS CloudFormation templates. It deploys your AWS -CloudFormation template in multiple AWS Regions and generates a report with a pass/fail -grade for each region. You can specify the regions and number of Availability Zones you -want to include in the test, and pass in parameter values from your AWS CloudFormation -template. TaskCat is implemented as a Python class that you import, instantiate, and run. - -TaskCat was developed by the aws-ia team to test AWS CloudFormation templates -that automatically deploy workloads on AWS. We’re pleased to make the tool available to -all developers who want to validate their custom AWS CloudFormation templates across -AWS Regions - -__See [TaskCat documentation](https://aws-ia.github.io/taskcat/).__ - -## Support -[![Feature Request](https://img.shields.io/badge/Open%20Issues-Feature%20Request-green.svg)](https://github.com/aws-ia/taskcat/issues/new/choose) -[![Report Bugs](https://img.shields.io/badge/Open%20Issue-Report%20Bug-red.svg)](https://github.com/aws-ia/taskcat/issues/new/choose) - -## GitHub - -[![GitHub stars](https://img.shields.io/github/stars/aws-ia/taskcat.svg?style=social&label=Stars)](https://github.com/aws-ia/taskcat) -[![GitHub issues](https://img.shields.io/github/issues/aws-ia/taskcat.svg)](https://github.com/aws-ia/taskcat/issues) -[![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/aws-ia/taskcat.svg)](https://github.com/aws-ia/taskcat/issues?q=is%3Aissue+is%3Aclosed) -[![GitHub pull requests](https://img.shields.io/github/issues-pr/aws-ia/taskcat.svg)](https://github.com/aws-ia/taskcat/pulls) -[![GitHub closed pull requests](https://img.shields.io/github/issues-pr-closed-raw/aws-ia/taskcat.svg)](https://github.com/aws-ia/taskcat/pulls?q=is%3Apr+is%3Aclosed) - -## PyPi - -[![PyPI - Downloads](https://img.shields.io/pypi/dw/taskcat.svg)](https://pypi.org/project/taskcat/#history) -[![PyPI - Downloads](https://img.shields.io/pypi/dm/taskcat.svg)](https://pypi.org/project/taskcat/#history) diff --git a/do_docs_generate.sh b/do_docs_generate.sh deleted file mode 100755 index 3640a2a31..000000000 --- a/do_docs_generate.sh +++ /dev/null @@ -1,18 +0,0 @@ -# Clean up docs -mkdir -p docs/apidocs -mkdir -p docs/schema -# Install docs tools -pip install portray pdocs json-schema-for-humans - -# Gererate API docs -portray as_html taskcat -o docs/apidocs/ --overwrite - -# Generate taskcat schema docs -python3 generate_schema.py -generate-schema-doc --config expand_buttons=true taskcat/cfg/config_schema.json docs/schema/taskcat_schema.md - -printf "\n\nReformatting schema files to specifications. Ignore the end-of-file-fixer error.\n\n" -pre-commit run --all-files - -# Push to gh_pages -portray on_github_pages --overwrite diff --git a/docs-requirements.txt b/docs-requirements.txt new file mode 100644 index 000000000..ed8e46e06 --- /dev/null +++ b/docs-requirements.txt @@ -0,0 +1,11 @@ +mkdocs>=1.5.0 +mkdocs-material>=9.4.0 +mkdocstrings[python]>=0.23.0 +mkdocs-gen-files>=0.5.0 +mkdocs-literate-nav>=0.6.0 +mkdocs-section-index>=0.3.0 +mkdocs-minify-plugin>=0.7.0 +mkdocs-git-revision-date-localized-plugin>=1.2.0 +pymdown-extensions>=10.0.0 +pillow>=10.0.0 +cairosvg>=2.7.0 diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md deleted file mode 100644 index 1826dedae..000000000 --- a/docs/INSTALLATION.md +++ /dev/null @@ -1,56 +0,0 @@ - -## Installation - -Currently only installation via pip is supported. - -### Requirements -![Python pip](https://img.shields.io/badge/Prerequisites-pip-blue.svg) -[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/taskcat.svg)](https://pypi.org/project/taskcat/#history) -![Python pip](https://img.shields.io/badge/Prerequisites-docker-yellow.svg) - -The host taskcat is run on requires access to an AWS account, this can be done by any -of the following mechanisms: - -1. Environment variables -2. Shared credential file (~/.aws/credentials) -3. AWS config file (~/.aws/config) -4. Assume Role provider -5. Boto2 config file (/etc/boto.cfg and ~/.boto) -6. Instance metadata service on an Amazon EC2 instance that has an IAM role configured. - -for more info see the [boto3 credential configuration documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html). - -!!! note - docker is only required if building lambda functions using a Dockerfile - -### Installing via pip3 - -```python -pip3 install taskcat -``` -### Installing via pip3 --user -*will install taskcat into homedir, useful if you get permissions errors with the regular method* - -```python -pip3 install taskcat --user -``` - -???+note - The user install dir is platform specific - - On Mac: - - - `~/Library/Python/3.x/bin/taskcat` - - On Linux: - - - `~/.local/bin` - -!!! warning - Be sure to add the python bin dir to your **$PATH** - -### Windows - -Taskcat on Windows is **not supported**. - -If you are running Windows 10 we recommend that you install [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/about) and then install taskcat inside the WSL environment. For details, see [Install and configure TaskCat on Microsoft Windows 10](https://aws.amazon.com/blogs/infrastructure-and-automation/install-and-configure-taskcat-on-microsoft-windows-10/). diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 27992a063..000000000 --- a/docs/README.md +++ /dev/null @@ -1,34 +0,0 @@ -[![logo](https://raw.githubusercontent.com/aws-ia/taskcat/main/assets/docs/images/tcat.png)](https://taskcat.io/) - -[![Build Status](https://travis-ci.com/aws-ia/taskcat.svg?branch=main)](https://travis-ci.com/aws-ia/taskcat) [![PyPI version](https://badge.fury.io/py/taskcat.svg)](https://badge.fury.io/py/taskcat) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) - - -## What is taskcat? -**taskcat** is a tool that tests AWS CloudFormation templates. It deploys your AWS -CloudFormation template in multiple AWS Regions and generates a report with a pass/fail -grade for each region. You can specify the regions and number of Availability Zones you -want to include in the test, and pass in parameter values from your AWS CloudFormation -template. taskcat is implemented as a Python class that you import, instantiate, and run. - -taskcat was developed by the aws-ia team to test AWS CloudFormation templates -that automatically deploy workloads on AWS. We’re pleased to make the tool available to -all developers who want to validate their custom AWS CloudFormation templates across -AWS Regions - - -## Support -[![Feature Request](https://img.shields.io/badge/Open%20Issues-Feature%20Request-green.svg)](https://github.com/aws-ia/taskcat/issues/new/choose) -[![Report Bugs](https://img.shields.io/badge/Open%20Issue-Report%20Bug-red.svg)](https://github.com/aws-ia/taskcat/issues/new/choose) - -## GitHub - -[![GitHub stars](https://img.shields.io/github/stars/aws-ia/taskcat.svg?style=social&label=Stars)](https://github.com/aws-ia/taskcat) -[![GitHub issues](https://img.shields.io/github/issues/aws-ia/taskcat.svg)](https://github.com/aws-ia/taskcat/issues) -[![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/aws-ia/taskcat.svg)](https://github.com/aws-ia/taskcat/issues?q=is%3Aissue+is%3Aclosed) -[![GitHub pull requests](https://img.shields.io/github/issues-pr/aws-ia/taskcat.svg)](https://github.com/aws-ia/taskcat/pulls) -[![GitHub closed pull requests](https://img.shields.io/github/issues-pr-closed-raw/aws-ia/taskcat.svg)](https://github.com/aws-ia/taskcat/pulls?q=is%3Apr+is%3Aclosed) - -## PyPi - -[![PyPI - Downloads](https://img.shields.io/pypi/dw/taskcat.svg)](https://pypi.org/project/taskcat/#history) -[![PyPI - Downloads](https://img.shields.io/pypi/dm/taskcat.svg)](https://pypi.org/project/taskcat/#history) diff --git a/docs/administrative/CODE_OF_CONDUCT.md b/docs/administrative/CODE_OF_CONDUCT.md index 754616508..a325d924b 100644 --- a/docs/administrative/CODE_OF_CONDUCT.md +++ b/docs/administrative/CODE_OF_CONDUCT.md @@ -2,11 +2,11 @@ ## 1. Purpose -A primary goal of TaskCat is to be inclusive to the largest number of contributors, with the most varied and diverse backgrounds possible. As such, we are committed to providing a friendly, safe and welcoming environment for all, regardless of gender, sexual orientation, ability, ethnicity, socioeconomic status, and religion (or lack thereof). +A primary goal of taskcat is to be inclusive to the largest number of contributors, with the most varied and diverse backgrounds possible. As such, we are committed to providing a friendly, safe and welcoming environment for all, regardless of gender, sexual orientation, ability, ethnicity, socioeconomic status, and religion (or lack thereof). This code of conduct outlines our expectations for all those who participate in our community, as well as the consequences for unacceptable behavior. -We invite all those who participate in TaskCat to help us create safe and positive experiences for everyone. +We invite all those who participate in taskcat to help us create safe and positive experiences for everyone. ## 2. Open Source Citizenship diff --git a/docs/administrative/CONTRIBUTING.md b/docs/administrative/CONTRIBUTING.md deleted file mode 100644 index 93ccf61b0..000000000 --- a/docs/administrative/CONTRIBUTING.md +++ /dev/null @@ -1,43 +0,0 @@ -# Contributing Guidelines - -Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community. - -Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. - -Documentation Links: - -[Module Documentation](https://aws-ia.github.io/taskcat/) - -[User Guide](https://aws-ia.github.io/auto-testing.html) - -## Reporting Bugs/Feature Requests - -We welcome you to use the GitHub issue tracker to report bugs or suggest features. - -When filing an issue, please check [existing open](https://github.com/aws-ia/taskcat/issues), or [recently closed](https://github.com/aws-ia/taskcat/issues?q=is%3Aissue+is%3Aclosed), issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: - -* A reproducible test case or series of steps -* The version of our code being used -* Any modifications you've made relevant to the bug -* Anything unusual about your environment or deployment - -## Contributing via Pull Requests (Pull request template provided) -Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: - -1. All changes are staged into the *develop* branch (Send PR to the *develop* branch) -2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. -3. You open an issue to discuss any significant work - we would hate for your time to be wasted. - -To send us a pull request, please: - -1. Fork the repository. -2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. -3. Ensure local tests pass. -4. Commit to your fork using clear commit messages. -5. Send us a pull request, answering any default questions in the pull request interface. -6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. - -GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). - -## Licensing -We may ask you to affirm the Apache 2.0 agreement for larger changes. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 000000000..2c31d1267 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,425 @@ +# API Reference + +Complete reference for taskcat's Python API and command-line interface. + +## Command Line Interface + +### Main Commands + +#### `taskcat test run` + +Execute taskcat tests with specified configuration. + +```bash +taskcat test run [OPTIONS] +``` + +**Options:** +- `--config-file, -c` - Path to configuration file (default: `.taskcat.yml`) +- `--regions` - Comma-separated list of regions to test +- `--tests` - Comma-separated list of tests to run +- `--no-delete` - Skip resource cleanup (for debugging) +- `--project-root` - Path to project root directory +- `--output-directory` - Directory for test outputs + +**Examples:** +```bash +# Run all tests +taskcat test run + +# Run specific test +taskcat test run --tests vpc-test + +# Run in specific regions +taskcat test run --regions us-east-1,us-west-2 + +# Keep resources for debugging +taskcat test run --no-delete +``` + +#### `taskcat lint` + +Validate taskcat configuration and CloudFormation templates. + +```bash +taskcat lint [OPTIONS] +``` + +**Options:** +- `--config-file, -c` - Path to configuration file +- `--templates` - Validate CloudFormation templates only +- `--strict` - Enable strict validation mode + +**Examples:** +```bash +# Lint configuration +taskcat lint + +# Lint specific file +taskcat lint -c custom.yml + +# Validate templates only +taskcat lint --templates +``` + +#### `taskcat test list` + +List available tests in configuration. + +```bash +taskcat test list [OPTIONS] +``` + +**Options:** +- `--config-file, -c` - Path to configuration file + +#### `taskcat upload` + +Upload templates and artifacts to S3. + +```bash +taskcat upload [OPTIONS] +``` + +**Options:** +- `--config-file, -c` - Path to configuration file +- `--bucket` - S3 bucket name +- `--key-prefix` - S3 key prefix + +### Global Options + +Available for all commands: + +- `--help, -h` - Show help message +- `--version` - Show version information +- `--debug` - Enable debug logging +- `--quiet, -q` - Suppress output + +## Python API + +### Core Classes + +#### `TaskCat` + +Main class for programmatic access to taskcat functionality. + +```python +from taskcat import TaskCat + +# Initialize TaskCat +tc = TaskCat( + config_file='.taskcat.yml', + project_root='/path/to/project', + regions=['us-east-1', 'us-west-2'] +) + +# Run tests +results = tc.test() + +# Get test results +for test_name, result in results.items(): + print(f"Test {test_name}: {result.status}") +``` + +#### `Config` + +Configuration management class. + +```python +from taskcat.config import Config + +# Load configuration +config = Config.create( + project_root='/path/to/project', + config_file='.taskcat.yml' +) + +# Access configuration +print(config.project.name) +print(config.tests.keys()) +``` + +#### `TestResult` + +Test execution result container. + +```python +# Access test results +result = tc.test()['test-name'] + +print(result.status) # PASS, FAIL, or ERROR +print(result.region) # AWS region +print(result.stack_name) # CloudFormation stack name +print(result.outputs) # Stack outputs +print(result.events) # CloudFormation events +``` + +### Configuration Objects + +#### `ProjectConfig` + +Project-level configuration. + +```python +project = config.project + +print(project.name) # Project name +print(project.regions) # Default regions +print(project.parameters) # Default parameters +print(project.s3_bucket) # S3 bucket +print(project.lambda_source_path) # Lambda source path +``` + +#### `TestConfig` + +Individual test configuration. + +```python +test = config.tests['test-name'] + +print(test.template) # Template path +print(test.parameters) # Test parameters +print(test.regions) # Test regions +print(test.auth) # Authentication settings +``` + +### Utility Functions + +#### Parameter Generation + +```python +from taskcat._template_params import ParamGen + +# Generate parameters +param_gen = ParamGen( + project_root='/path/to/project', + param_dict={'Password': '$[taskcat_genpass_16S]'}, + bucket_name='my-bucket', + region='us-east-1', + boto_client=boto3.client('cloudformation'), + project_name='my-project', + test_name='my-test' +) + +# Access generated parameters +generated_params = param_gen.results +print(generated_params['Password']) # Generated password +``` + +#### Template Processing + +```python +from taskcat._cfn_lint import CfnLint + +# Validate CloudFormation template +linter = CfnLint() +results = linter.lint_file('template.yaml') + +for result in results: + print(f"{result.level}: {result.message}") +``` + +### Exception Handling + +#### `TaskCatException` + +Base exception for taskcat errors. + +```python +from taskcat.exceptions import TaskCatException + +try: + tc = TaskCat(config_file='invalid.yml') + results = tc.test() +except TaskCatException as e: + print(f"TaskCat error: {e}") +``` + +#### Common Exceptions + +- `TaskCatException` - Base taskcat exception +- `ConfigError` - Configuration validation errors +- `TemplateError` - CloudFormation template errors +- `RegionError` - AWS region-related errors + +### Advanced Usage + +#### Custom Hooks + +```python +from taskcat import TaskCat + +class CustomTaskCat(TaskCat): + def pre_test_hook(self, test_name, region): + """Execute before each test""" + print(f"Starting test {test_name} in {region}") + + def post_test_hook(self, test_name, region, result): + """Execute after each test""" + print(f"Test {test_name} completed: {result.status}") + +# Use custom class +tc = CustomTaskCat(config_file='.taskcat.yml') +results = tc.test() +``` + +#### Parallel Execution + +```python +import concurrent.futures +from taskcat import TaskCat + +def run_test(test_config): + tc = TaskCat(config_file=test_config) + return tc.test() + +# Run tests in parallel +configs = ['test1.yml', 'test2.yml', 'test3.yml'] + +with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [executor.submit(run_test, config) for config in configs] + results = [future.result() for future in futures] +``` + +#### Custom Parameter Generation + +```python +from taskcat._template_params import ParamGen + +class CustomParamGen(ParamGen): + def custom_function(self): + """Custom parameter generation function""" + return "custom-value" + + def transform_parameter(self): + # Call parent method + super().transform_parameter() + + # Add custom transformations + if '$[custom_function]' in self.param_value: + self.param_value = self.param_value.replace( + '$[custom_function]', + self.custom_function() + ) +``` + +## Integration Examples + +### CI/CD Integration + +```python +#!/usr/bin/env python3 +"""CI/CD integration script""" + +import sys +from taskcat import TaskCat +from taskcat.exceptions import TaskCatException + +def main(): + try: + # Initialize TaskCat + tc = TaskCat( + config_file='.taskcat.yml', + regions=['us-east-1', 'us-west-2'] + ) + + # Run tests + results = tc.test() + + # Check results + failed_tests = [ + name for name, result in results.items() + if result.status != 'PASS' + ] + + if failed_tests: + print(f"Failed tests: {failed_tests}") + sys.exit(1) + else: + print("All tests passed!") + sys.exit(0) + + except TaskCatException as e: + print(f"TaskCat error: {e}") + sys.exit(1) + +if __name__ == '__main__': + main() +``` + +### Custom Reporting + +```python +import json +from taskcat import TaskCat + +def generate_custom_report(results): + """Generate custom test report""" + report = { + 'summary': { + 'total_tests': len(results), + 'passed': sum(1 for r in results.values() if r.status == 'PASS'), + 'failed': sum(1 for r in results.values() if r.status == 'FAIL'), + }, + 'details': [] + } + + for test_name, result in results.items(): + report['details'].append({ + 'test_name': test_name, + 'status': result.status, + 'region': result.region, + 'stack_name': result.stack_name, + 'duration': result.duration, + 'outputs': result.outputs + }) + + return report + +# Run tests and generate report +tc = TaskCat(config_file='.taskcat.yml') +results = tc.test() +report = generate_custom_report(results) + +# Save report +with open('test-report.json', 'w') as f: + json.dump(report, f, indent=2) +``` + +## Environment Variables + +TaskCat recognizes these environment variables: + +- `AWS_PROFILE` - AWS profile to use +- `AWS_REGION` - Default AWS region +- `TASKCAT_CONFIG_FILE` - Default configuration file path +- `TASKCAT_PROJECT_ROOT` - Default project root directory +- `TASKCAT_DEBUG` - Enable debug logging (set to `1`) + +## Return Codes + +Command-line return codes: + +- `0` - Success +- `1` - General error +- `2` - Configuration error +- `3` - Template validation error +- `4` - Test execution error + +## Version Information + +```bash +# Get version +taskcat --version + +# Get detailed version info +taskcat --version --verbose +``` + +```python +# Get version programmatically +import taskcat +print(taskcat.__version__) +``` + +For more detailed API documentation, see the inline docstrings and type hints in the source code. diff --git a/docs/assets/css/cyborg-theme.css b/docs/assets/css/cyborg-theme.css new file mode 100644 index 000000000..f39572083 --- /dev/null +++ b/docs/assets/css/cyborg-theme.css @@ -0,0 +1,319 @@ +/* Cyborg Theme Customizations for taskcat */ + +:root { + /* Cyborg theme colors */ + --cyborg-bg: #060606; + --cyborg-bg-alt: #121212; + --cyborg-text: #fff; + --cyborg-text-muted: #888; + --cyborg-primary: #2a9fd6; + --cyborg-secondary: #555; + --cyborg-success: #77b300; + --cyborg-info: #9933cc; + --cyborg-warning: #ff8800; + --cyborg-danger: #cc0000; + --cyborg-border: #222; + --cyborg-border-light: #333; + --cyborg-code-bg: #222; + --cyborg-code-text: #0f0; +} + +/* Base overrides */ +body { + background-color: var(--cyborg-bg); + color: var(--cyborg-text); +} + +/* Header styling */ +.md-header { + background-color: var(--cyborg-bg-alt); + color: var(--cyborg-text); + border-bottom: 1px solid var(--cyborg-border); +} + +.md-header__title { + color: var(--cyborg-primary); + font-weight: 700; +} + +/* Navigation styling */ +.md-nav { + color: var(--cyborg-text); +} + +.md-nav__title { + color: var(--cyborg-primary); + font-weight: 700; +} + +.md-nav__link { + color: var(--cyborg-text) !important; +} + +.md-nav__link:hover { + color: var(--cyborg-primary) !important; +} + +.md-nav__link--active { + color: var(--cyborg-primary) !important; + font-weight: 700; +} + +/* Sidebar styling */ +.md-sidebar { + background-color: var(--cyborg-bg-alt); + border-right: 1px solid var(--cyborg-border); +} + +/* Typography */ +h1 { + color: var(--cyborg-primary) !important; + border-bottom: 2px solid var(--cyborg-primary); + padding-bottom: 0.5rem; +} + +h2 { + color: var(--cyborg-info) !important; +} + +h3 { + color: var(--cyborg-warning) !important; +} + +h4 { + color: var(--cyborg-success) !important; +} + +a { + color: var(--cyborg-primary) !important; +} + +a:hover { + color: var(--cyborg-info) !important; + text-decoration: underline; +} + +/* Code blocks */ +.highlight { + background-color: var(--cyborg-code-bg) !important; + border: 1px solid var(--cyborg-border-light); +} + +.highlight pre { + color: var(--cyborg-text) !important; +} + +/* Inline code */ +code { + background-color: var(--cyborg-code-bg) !important; + color: var(--cyborg-code-text) !important; + border: 1px solid var(--cyborg-border-light); + padding: 0.2em 0.4em; +} + +/* Syntax highlighting - Cyborg style */ +.highlight .k { color: #cc99cd !important; } /* Keyword */ +.highlight .s { color: #8fc13e !important; } /* String */ +.highlight .c { color: #999999 !important; font-style: italic; } /* Comment */ +.highlight .m { color: #f08d49 !important; } /* Number */ +.highlight .o { color: #f8f8f2 !important; } /* Operator */ +.highlight .p { color: #f8f8f2 !important; } /* Punctuation */ +.highlight .n { color: #f8f8f2 !important; } /* Name */ +.highlight .nf { color: #2a9fd6 !important; } /* Function name */ +.highlight .nb { color: #f08d49 !important; } /* Built-in */ + +/* Tables */ +table { + background-color: var(--cyborg-bg-alt) !important; + border: 1px solid var(--cyborg-border) !important; +} + +th { + background-color: var(--cyborg-bg-alt) !important; + color: var(--cyborg-primary) !important; + border-bottom: 2px solid var(--cyborg-primary) !important; +} + +td { + border-bottom: 1px solid var(--cyborg-border) !important; +} + +tr:hover { + background-color: var(--cyborg-bg) !important; +} + +/* Admonitions */ +.admonition { + background-color: var(--cyborg-bg-alt) !important; + border-left: 4px solid var(--cyborg-primary) !important; +} + +.admonition-title { + background-color: rgba(42, 159, 214, 0.1) !important; + color: var(--cyborg-primary) !important; +} + +.admonition.note { + border-left-color: var(--cyborg-info) !important; +} + +.admonition.note .admonition-title { + background-color: rgba(153, 51, 204, 0.1) !important; + color: var(--cyborg-info) !important; +} + +.admonition.warning { + border-left-color: var(--cyborg-warning) !important; +} + +.admonition.warning .admonition-title { + background-color: rgba(255, 136, 0, 0.1) !important; + color: var(--cyborg-warning) !important; +} + +.admonition.danger { + border-left-color: var(--cyborg-danger) !important; +} + +.admonition.danger .admonition-title { + background-color: rgba(204, 0, 0, 0.1) !important; + color: var(--cyborg-danger) !important; +} + +/* Buttons */ +.md-button { + background-color: var(--cyborg-primary) !important; + color: var(--cyborg-text) !important; + border: none !important; +} + +.md-button:hover { + background-color: var(--cyborg-info) !important; + color: var(--cyborg-text) !important; + transform: translateY(-1px); +} + +/* Search */ +.md-search__input { + background-color: var(--cyborg-bg) !important; + color: var(--cyborg-text) !important; + border: 1px solid var(--cyborg-border-light) !important; +} + +.md-search__input::placeholder { + color: var(--cyborg-text-muted) !important; +} + +/* Footer */ +.md-footer { + background-color: var(--cyborg-bg-alt) !important; + color: var(--cyborg-text-muted) !important; +} + +.md-footer-meta { + background-color: var(--cyborg-bg) !important; +} + +/* Tabs */ +.tabbed-set > .tabbed-labels { + background-color: var(--cyborg-bg-alt) !important; +} + +.tabbed-set > .tabbed-labels > label { + color: var(--cyborg-text) !important; + background-color: var(--cyborg-bg-alt) !important; +} + +.tabbed-set > .tabbed-labels > label:hover { + color: var(--cyborg-primary) !important; +} + +.tabbed-set > .tabbed-labels > label[for]:checked { + color: var(--cyborg-primary) !important; + border-bottom: 2px solid var(--cyborg-primary) !important; +} + +/* Copy code button */ +.md-clipboard { + color: var(--cyborg-text-muted) !important; +} + +.md-clipboard:hover { + color: var(--cyborg-primary) !important; +} + +/* Custom elements */ +.feature-card { + background-color: var(--cyborg-bg-alt); + border: 1px solid var(--cyborg-border); + padding: 1.5rem; + margin-bottom: 1.5rem; + transition: all 0.3s ease; +} + +.feature-card:hover { + border-color: var(--cyborg-primary); + box-shadow: 0 0 10px rgba(42, 159, 214, 0.3); + transform: translateY(-5px); +} + +.feature-card h3 { + color: var(--cyborg-primary) !important; + margin-top: 0; +} + +/* Schema styling */ +.schema-container { + background-color: var(--cyborg-bg-alt); + border: 1px solid var(--cyborg-border); + margin: 1rem 0; + padding: 1rem; +} + +.schema-property { + margin: 1rem 0; + padding: 1rem; + background-color: var(--cyborg-bg); + border-left: 4px solid var(--cyborg-primary); +} + +.schema-property-name { + color: var(--cyborg-primary); + font-weight: 700; + font-family: 'Roboto Mono', monospace; +} + +.schema-property-type { + color: var(--cyborg-warning); + font-size: 0.9rem; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .md-nav__title { + background-color: var(--cyborg-bg-alt) !important; + color: var(--cyborg-primary) !important; + } +} + +/* Print styles */ +@media print { + body { + background-color: white !important; + color: black !important; + } + + h1, h2, h3, h4, h5, h6 { + color: black !important; + } + + a { + color: blue !important; + } + + pre, code { + background-color: #f5f5f5 !important; + color: black !important; + border: 1px solid #ddd !important; + } +} diff --git a/docs/assets/images/tcat.png b/docs/assets/images/tcat.png new file mode 100644 index 000000000..f50516cf8 Binary files /dev/null and b/docs/assets/images/tcat.png differ diff --git a/docs/assets/js/cyborg-theme.js b/docs/assets/js/cyborg-theme.js new file mode 100644 index 000000000..726801222 --- /dev/null +++ b/docs/assets/js/cyborg-theme.js @@ -0,0 +1,344 @@ +// Cyborg Theme Enhancements for taskcat Documentation + +document.addEventListener('DOMContentLoaded', function() { + // Initialize Cyborg theme enhancements + initCyborgTheme(); + addCodeCopyButtons(); + enhanceNavigation(); + addKeyboardShortcuts(); + addNeonEffects(); +}); + +// Initialize Cyborg theme +function initCyborgTheme() { + // Add theme class to body + document.body.classList.add('cyborg-theme'); + + // Add theme version to footer + const footer = document.querySelector('.md-footer-copyright'); + if (footer) { + const themeInfo = document.createElement('div'); + themeInfo.className = 'md-footer-theme-info'; + themeInfo.innerHTML = 'Powered by Cyborg'; + footer.appendChild(themeInfo); + } +} + +// Add copy buttons to code blocks +function addCodeCopyButtons() { + const codeBlocks = document.querySelectorAll('.highlight pre'); + + codeBlocks.forEach(function(codeBlock) { + const wrapper = document.createElement('div'); + wrapper.className = 'code-wrapper'; + wrapper.style.position = 'relative'; + + codeBlock.parentNode.insertBefore(wrapper, codeBlock); + wrapper.appendChild(codeBlock); + + const copyButton = document.createElement('button'); + copyButton.className = 'copy-button'; + copyButton.innerHTML = '⚡ Copy'; + copyButton.setAttribute('aria-label', 'Copy code to clipboard'); + copyButton.style.cssText = ` + position: absolute; + top: 5px; + right: 5px; + background: #2a9fd6; + color: #fff; + border: none; + border-radius: 3px; + padding: 5px 10px; + font-size: 12px; + cursor: pointer; + opacity: 0.8; + transition: all 0.3s ease; + z-index: 10; + `; + + copyButton.addEventListener('mouseenter', function() { + this.style.opacity = '1'; + this.style.boxShadow = '0 0 10px rgba(42, 159, 214, 0.7)'; + }); + + copyButton.addEventListener('mouseleave', function() { + this.style.opacity = '0.8'; + this.style.boxShadow = 'none'; + }); + + copyButton.addEventListener('click', function() { + const code = codeBlock.textContent; + + if (navigator.clipboard && window.isSecureContext) { + navigator.clipboard.writeText(code).then(function() { + showCopySuccess(copyButton); + }).catch(function() { + fallbackCopy(code, copyButton); + }); + } else { + fallbackCopy(code, copyButton); + } + }); + + wrapper.appendChild(copyButton); + }); +} + +// Fallback copy method for older browsers +function fallbackCopy(text, button) { + const textArea = document.createElement('textarea'); + textArea.value = text; + textArea.style.position = 'fixed'; + textArea.style.left = '-999999px'; + textArea.style.top = '-999999px'; + document.body.appendChild(textArea); + textArea.focus(); + textArea.select(); + + try { + document.execCommand('copy'); + showCopySuccess(button); + } catch (err) { + console.error('Failed to copy text: ', err); + button.innerHTML = '❌ Failed'; + button.style.background = '#cc0000'; + setTimeout(function() { + button.innerHTML = '⚡ Copy'; + button.style.background = '#2a9fd6'; + }, 2000); + } + + document.body.removeChild(textArea); +} + +// Show copy success feedback +function showCopySuccess(button) { + const originalText = button.innerHTML; + button.innerHTML = '✓ Copied!'; + button.style.background = '#77b300'; + button.style.boxShadow = '0 0 10px rgba(119, 179, 0, 0.7)'; + + setTimeout(function() { + button.innerHTML = originalText; + button.style.background = '#2a9fd6'; + button.style.boxShadow = '0 0 10px rgba(42, 159, 214, 0.7)'; + }, 2000); +} + +// Enhance navigation +function enhanceNavigation() { + // Add active class to current page in navigation + const currentPath = window.location.pathname; + const navLinks = document.querySelectorAll('.md-nav__link'); + + navLinks.forEach(function(link) { + const href = link.getAttribute('href'); + if (href && currentPath.endsWith(href)) { + link.classList.add('md-nav__link--active'); + link.style.borderLeft = '3px solid #2a9fd6'; + link.style.paddingLeft = '10px'; + + // Expand parent sections + let parent = link.parentElement; + while (parent) { + if (parent.classList.contains('md-nav__item--nested')) { + parent.classList.add('md-nav__item--expanded'); + const input = parent.querySelector('input'); + if (input) { + input.checked = true; + } + } + parent = parent.parentElement; + } + } + }); + + // Add hover effects to navigation links + navLinks.forEach(function(link) { + link.addEventListener('mouseenter', function() { + if (!this.classList.contains('md-nav__link--active')) { + this.style.borderLeft = '3px solid #9933cc'; + this.style.paddingLeft = '10px'; + this.style.transition = 'all 0.3s ease'; + } + }); + + link.addEventListener('mouseleave', function() { + if (!this.classList.contains('md-nav__link--active')) { + this.style.borderLeft = 'none'; + this.style.paddingLeft = '0'; + } + }); + }); +} + +// Add keyboard shortcuts +function addKeyboardShortcuts() { + document.addEventListener('keydown', function(e) { + // Ctrl/Cmd + K for search + if ((e.ctrlKey || e.metaKey) && e.key === 'k') { + e.preventDefault(); + const searchInput = document.querySelector('.md-search__input'); + if (searchInput) { + searchInput.focus(); + } + } + + // Escape to close search + if (e.key === 'Escape') { + const searchInput = document.querySelector('.md-search__input'); + if (searchInput && document.activeElement === searchInput) { + searchInput.blur(); + } + } + }); +} + +// Add neon effects for Cyborg theme +function addNeonEffects() { + // Add neon effect to headings + const headings = document.querySelectorAll('h1, h2, h3'); + headings.forEach(function(heading) { + heading.addEventListener('mouseenter', function() { + this.style.textShadow = '0 0 10px currentColor'; + this.style.transition = 'text-shadow 0.3s ease'; + }); + + heading.addEventListener('mouseleave', function() { + this.style.textShadow = 'none'; + }); + }); + + // Add neon effect to code blocks + const codeBlocks = document.querySelectorAll('.highlight'); + codeBlocks.forEach(function(block) { + block.addEventListener('mouseenter', function() { + this.style.boxShadow = '0 0 15px rgba(42, 159, 214, 0.5)'; + this.style.transition = 'box-shadow 0.3s ease'; + }); + + block.addEventListener('mouseleave', function() { + this.style.boxShadow = 'none'; + }); + }); + + // Add neon effect to buttons + const buttons = document.querySelectorAll('.md-button'); + buttons.forEach(function(button) { + button.addEventListener('mouseenter', function() { + this.style.boxShadow = '0 0 15px rgba(42, 159, 214, 0.7)'; + this.style.transition = 'all 0.3s ease'; + }); + + button.addEventListener('mouseleave', function() { + this.style.boxShadow = 'none'; + }); + }); +} + +// Add smooth scrolling for anchor links +function addSmoothScrolling() { + document.querySelectorAll('a[href^="#"]').forEach(function(anchor) { + anchor.addEventListener('click', function(e) { + const targetId = this.getAttribute('href').substring(1); + const targetElement = document.getElementById(targetId); + + if (targetElement) { + e.preventDefault(); + targetElement.scrollIntoView({ + behavior: 'smooth', + block: 'start' + }); + + // Update URL without jumping + history.pushState(null, null, '#' + targetId); + } + }); + }); +} + +// Initialize smooth scrolling +setTimeout(addSmoothScrolling, 1000); + +// Add accessibility features +function addAccessibilityFeatures() { + // Add skip to content link + const skipLink = document.createElement('a'); + skipLink.href = '#main-content'; + skipLink.textContent = 'Skip to main content'; + skipLink.className = 'skip-link'; + skipLink.style.cssText = ` + position: absolute; + top: -40px; + left: 0; + background: #2a9fd6; + color: #fff; + padding: 8px; + z-index: 1000; + transition: top 0.3s; + `; + + skipLink.addEventListener('focus', function() { + this.style.top = '0'; + }); + + skipLink.addEventListener('blur', function() { + this.style.top = '-40px'; + }); + + document.body.insertBefore(skipLink, document.body.firstChild); + + // Add main content landmark + const content = document.querySelector('.md-content'); + if (content) { + content.setAttribute('id', 'main-content'); + content.setAttribute('role', 'main'); + } +} + +// Initialize accessibility features +setTimeout(addAccessibilityFeatures, 500); + +// Add print styles +function addPrintStyles() { + const printStyles = document.createElement('style'); + printStyles.textContent = ` + @media print { + .md-header, .md-tabs, .md-sidebar, .md-footer, .md-top, .md-dialog, .md-search { + display: none !important; + } + + .md-content { + margin: 0 !important; + max-width: 100% !important; + } + + .md-content__inner { + padding: 0 !important; + margin: 0 !important; + } + + body { + color: black !important; + background: white !important; + } + + h1, h2, h3, h4, h5, h6 { + color: black !important; + break-after: avoid !important; + } + + pre, blockquote, tr, img { + break-inside: avoid !important; + } + + .copy-button { + display: none !important; + } + } + `; + document.head.appendChild(printStyles); +} + +// Initialize print styles +addPrintStyles(); diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..35aab67c5 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,478 @@ +# Configuration + +Learn how to configure taskcat for your specific testing needs with comprehensive configuration options. + +## Configuration File Structure + +taskcat uses YAML configuration files (`.taskcat.yml`) with this structure: + +```yaml +general: + # Global settings applied to all tests + +project: + # Project-specific settings + +tests: + # Individual test definitions +``` + +## Basic Configuration + +### Minimal Configuration + +```yaml +project: + name: my-project + regions: + - us-east-1 + +tests: + basic: + template: template.yaml +``` + +### Standard Configuration + +```yaml +project: + name: my-cloudformation-project + regions: + - us-east-1 + - us-west-2 + parameters: + Environment: test + Owner: development-team + +tests: + vpc-test: + template: templates/vpc.yaml + parameters: + VpcCidr: 10.0.0.0/16 + + app-test: + template: templates/app.yaml + parameters: + InstanceType: t3.micro + DatabasePassword: $[taskcat_genpass_16S] +``` + +## Global Settings + +Configure settings that apply to all tests: + +```yaml +general: + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + parameters: + Environment: production + ProjectName: my-project + tags: + CostCenter: "1001" + Department: Engineering + s3_regional_buckets: true +``` + +### Global Properties + +| Property | Description | Example | +|----------|-------------|---------| +| `regions` | Default regions for all tests | `["us-east-1", "us-west-2"]` | +| `parameters` | Global CloudFormation parameters | `{"Environment": "prod"}` | +| `tags` | CloudFormation stack tags | `{"Project": "taskcat"}` | +| `s3_bucket` | Custom S3 bucket name | `"my-taskcat-bucket"` | +| `s3_regional_buckets` | Enable regional buckets | `true` | +| `auth` | AWS authentication profiles | `{"default": "my-profile"}` | + +## Project Settings + +Configure project-specific options: + +```yaml +project: + name: enterprise-application + owner: platform-team@company.com + regions: + - us-east-1 + - us-west-2 + parameters: + ProjectName: enterprise-app + Environment: production + lambda_source_path: functions/source + lambda_zip_path: functions/packages + package_lambda: true + shorten_stack_name: true + s3_object_acl: bucket-owner-read +``` + +### Project Properties + +| Property | Description | Default | +|----------|-------------|---------| +| `name` | Project identifier | Required | +| `owner` | Project owner email | - | +| `template` | Default template path | - | +| `lambda_source_path` | Lambda source directory | `lambda_functions/source` | +| `lambda_zip_path` | Lambda package directory | `lambda_functions/packages` | +| `package_lambda` | Enable Lambda packaging | `true` | +| `build_submodules` | Build submodule Lambdas | `true` | +| `shorten_stack_name` | Use short stack names | `false` | +| `role_name` | CloudFormation service role | - | + +## Test Configuration + +Define individual tests with specific settings: + +```yaml +tests: + # Basic test + simple-test: + template: templates/simple.yaml + + # Test with parameters + parameterized-test: + template: templates/app.yaml + parameters: + InstanceType: t3.medium + DatabasePassword: $[taskcat_genpass_20S] + + # Test with specific regions + regional-test: + template: templates/global.yaml + regions: + - us-east-1 + - eu-west-1 + - ap-southeast-1 + + # Test with authentication + authenticated-test: + template: templates/secure.yaml + auth: + us-east-1: production-profile + eu-west-1: europe-profile +``` + +### Test Properties + +| Property | Description | Required | +|----------|-------------|----------| +| `template` | CloudFormation template path | ✅ | +| `parameters` | Test-specific parameters | - | +| `regions` | Test-specific regions | - | +| `auth` | Authentication overrides | - | +| `artifact_regions` | Artifact copy regions | - | + +## Advanced Configuration + +### Multi-Environment Setup + +```yaml +project: + name: multi-env-app + +tests: + development: + template: templates/app.yaml + parameters: + Environment: dev + InstanceType: t3.micro + DatabaseInstanceClass: db.t3.micro + regions: + - us-east-1 + + staging: + template: templates/app.yaml + parameters: + Environment: staging + InstanceType: t3.small + DatabaseInstanceClass: db.t3.small + regions: + - us-east-1 + - us-west-2 + + production: + template: templates/app.yaml + parameters: + Environment: prod + InstanceType: m5.large + DatabaseInstanceClass: db.r5.large + regions: + - us-east-1 + - us-west-2 + - eu-west-1 +``` + +### Cross-Account Testing + +```yaml +project: + name: cross-account-app + +tests: + development-account: + template: templates/app.yaml + auth: + default: dev-account-profile + parameters: + Environment: dev + + production-account: + template: templates/app.yaml + auth: + default: prod-account-profile + parameters: + Environment: prod +``` + +### Lambda Function Testing + +```yaml +project: + name: serverless-app + lambda_source_path: src/functions + lambda_zip_path: dist/functions + package_lambda: true + build_submodules: true + +tests: + lambda-test: + template: templates/serverless.yaml + parameters: + Runtime: python3.9 + MemorySize: 256 + Timeout: 30 +``` + +## Authentication Configuration + +Configure AWS authentication for different regions or accounts: + +### Profile-Based Authentication + +```yaml +general: + auth: + default: my-default-profile + us-gov-east-1: govcloud-profile + cn-north-1: china-profile + +project: + auth: + us-east-1: production-profile + eu-west-1: europe-profile +``` + +### Test-Specific Authentication + +```yaml +tests: + secure-test: + template: templates/secure.yaml + auth: + us-east-1: security-profile + us-west-2: security-profile +``` + +## Parameter Configuration + +### Static Parameters + +```yaml +tests: + static-test: + template: templates/app.yaml + parameters: + InstanceType: t3.medium + Environment: production + EnableLogging: true + Port: 8080 +``` + +### Dynamic Parameters + +```yaml +tests: + dynamic-test: + template: templates/app.yaml + parameters: + # Generate unique values + S3Bucket: $[taskcat_autobucket] + DatabasePassword: $[taskcat_genpass_16S] + UniqueId: $[taskcat_genuuid] + + # Environment-aware values + CurrentRegion: $[taskcat_current_region] + ProjectName: $[taskcat_project_name] + TestName: $[taskcat_test_name] + + # AWS resource values + AvailabilityZones: $[taskcat_genaz_2] + KeyPair: $[taskcat_getkeypair] + + # External values + DatabaseHost: $[taskcat_ssm_/app/database/host] + ApiKey: $[taskcat_secretsmanager_prod/api/key] +``` + +## S3 Configuration + +### Bucket Management + +```yaml +project: + # Use custom bucket + s3_bucket: my-custom-taskcat-bucket + + # Enable regional buckets + s3_regional_buckets: true + + # Set object ACL + s3_object_acl: bucket-owner-read + + # Enable legacy signature version + s3_enable_sig_v2: false +``` + +### Artifact Regions + +```yaml +general: + artifact_regions: + - us-east-1 + - us-west-2 + - eu-west-1 + +tests: + global-test: + template: templates/global.yaml + artifact_regions: + - us-east-1 + - ap-southeast-1 +``` + +## Hooks Configuration + +Execute custom scripts before or after tests: + +```yaml +general: + prehooks: + - type: script + config: + command: ./scripts/setup.sh + + posthooks: + - type: script + config: + command: ./scripts/cleanup.sh + +tests: + custom-test: + template: templates/app.yaml + prehooks: + - type: script + config: + command: ./scripts/test-setup.sh +``` + +## Validation and Linting + +Validate your configuration: + +```bash +# Lint configuration +taskcat lint + +# Lint specific file +taskcat lint --config-file custom.yml + +# Validate templates +taskcat lint --templates +``` + +## Best Practices + +### 1. Use Hierarchical Configuration + +```yaml +# Global defaults +general: + regions: + - us-east-1 + - us-west-2 + parameters: + Environment: test + +# Project overrides +project: + parameters: + ProjectName: my-app + +# Test-specific settings +tests: + production: + parameters: + Environment: prod # Overrides global +``` + +### 2. Leverage Dynamic Values + +```yaml +tests: + flexible-test: + template: templates/app.yaml + parameters: + # Avoid hardcoded values + S3Bucket: $[taskcat_autobucket] + DatabasePassword: $[taskcat_genpass_20S] + + # Use context-aware values + StackName: $[taskcat_project_name]-$[taskcat_test_name] + Region: $[taskcat_current_region] +``` + +### 3. Organize Templates + +```yaml +project: + name: organized-project + +tests: + infrastructure: + template: infrastructure/vpc.yaml + + database: + template: database/rds.yaml + + application: + template: application/app.yaml +``` + +### 4. Use Meaningful Names + +```yaml +tests: + # Good: Descriptive names + vpc-with-public-subnets: + template: templates/vpc-public.yaml + + rds-mysql-multi-az: + template: templates/rds-mysql.yaml + + # Avoid: Generic names + test1: + template: template1.yaml +``` + +## Configuration Examples + +See the [Examples](examples.md) page for complete, real-world configuration examples. + +## Next Steps + +- [Dynamic Values](dynamic-values.md) - Master runtime parameters +- [Parameter Overrides](parameter-overrides.md) - Advanced parameter techniques +- [Schema Reference](schema.md) - Complete configuration reference diff --git a/docs/custom.css b/docs/custom.css deleted file mode 100644 index a870e2055..000000000 --- a/docs/custom.css +++ /dev/null @@ -1,95 +0,0 @@ -.md-typeset .example > .admonition-title, .md-typeset .example > summary { - /* background-color: rgba(124,77,255,.1); */ - /* border-color: #7c4dff; */ - background-color: #FFECB3; - border-color: orange; -} - -.md-typeset .example > .admonition-title::before, .md-typeset .example > summary::before { - /* background-color: #7c4dff; */ - background-color: black; -} - -.md-typeset .admonition.example, .md-typeset details.example { - /* border-color: #7c4dff; */ - border-color: orange; -} -[data-md-color-primary="black"] { - /* --md-typeset-a-color: #4051b5; */ - --md-typeset-a-color: orange; -} - -pre { line-height: 125%; } -td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } -span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } -td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } -span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } -.codehilite .hll { background-color: #ffffcc } -.codehilite { background: #f8f8f8; } -.codehilite .c { color: #408080; font-style: italic } /* Comment */ -.codehilite .err { border: 1px solid #FF0000 } /* Error */ -.codehilite .k { color: #008000; font-weight: bold } /* Keyword */ -.codehilite .o { color: #666666 } /* Operator */ -.codehilite .ch { color: #408080; font-style: italic } /* Comment.Hashbang */ -.codehilite .cm { color: #408080; font-style: italic } /* Comment.Multiline */ -.codehilite .cp { color: #BC7A00 } /* Comment.Preproc */ -.codehilite .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */ -.codehilite .c1 { color: #408080; font-style: italic } /* Comment.Single */ -.codehilite .cs { color: #408080; font-style: italic } /* Comment.Special */ -.codehilite .gd { color: #A00000 } /* Generic.Deleted */ -.codehilite .ge { font-style: italic } /* Generic.Emph */ -.codehilite .gr { color: #FF0000 } /* Generic.Error */ -.codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */ -.codehilite .gi { color: #00A000 } /* Generic.Inserted */ -.codehilite .go { color: #888888 } /* Generic.Output */ -.codehilite .gp { color: #000080; font-weight: bold } /* Generic.Prompt */ -.codehilite .gs { font-weight: bold } /* Generic.Strong */ -.codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ -.codehilite .gt { color: #0044DD } /* Generic.Traceback */ -.codehilite .kc { color: #008000; font-weight: bold } /* Keyword.Constant */ -.codehilite .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ -.codehilite .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ -.codehilite .kp { color: #008000 } /* Keyword.Pseudo */ -.codehilite .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ -.codehilite .kt { color: #B00040 } /* Keyword.Type */ -.codehilite .m { color: #666666 } /* Literal.Number */ -.codehilite .s { color: #BA2121 } /* Literal.String */ -.codehilite .na { color: #7D9029 } /* Name.Attribute */ -.codehilite .nb { color: #008000 } /* Name.Builtin */ -.codehilite .nc { color: #0000FF; font-weight: bold } /* Name.Class */ -.codehilite .no { color: #880000 } /* Name.Constant */ -.codehilite .nd { color: #AA22FF } /* Name.Decorator */ -.codehilite .ni { color: #999999; font-weight: bold } /* Name.Entity */ -.codehilite .ne { color: #D2413A; font-weight: bold } /* Name.Exception */ -.codehilite .nf { color: #0000FF } /* Name.Function */ -.codehilite .nl { color: #A0A000 } /* Name.Label */ -.codehilite .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ -.codehilite .nt { color: #008000; font-weight: bold } /* Name.Tag */ -.codehilite .nv { color: #19177C } /* Name.Variable */ -.codehilite .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ -.codehilite .w { color: #bbbbbb } /* Text.Whitespace */ -.codehilite .mb { color: #666666 } /* Literal.Number.Bin */ -.codehilite .mf { color: #666666 } /* Literal.Number.Float */ -.codehilite .mh { color: #666666 } /* Literal.Number.Hex */ -.codehilite .mi { color: #666666 } /* Literal.Number.Integer */ -.codehilite .mo { color: #666666 } /* Literal.Number.Oct */ -.codehilite .sa { color: #BA2121 } /* Literal.String.Affix */ -.codehilite .sb { color: #BA2121 } /* Literal.String.Backtick */ -.codehilite .sc { color: #BA2121 } /* Literal.String.Char */ -.codehilite .dl { color: #BA2121 } /* Literal.String.Delimiter */ -.codehilite .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ -.codehilite .s2 { color: #BA2121 } /* Literal.String.Double */ -.codehilite .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */ -.codehilite .sh { color: #BA2121 } /* Literal.String.Heredoc */ -.codehilite .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */ -.codehilite .sx { color: #008000 } /* Literal.String.Other */ -.codehilite .sr { color: #BB6688 } /* Literal.String.Regex */ -.codehilite .s1 { color: #BA2121 } /* Literal.String.Single */ -.codehilite .ss { color: #19177C } /* Literal.String.Symbol */ -.codehilite .bp { color: #008000 } /* Name.Builtin.Pseudo */ -.codehilite .fm { color: #0000FF } /* Name.Function.Magic */ -.codehilite .vc { color: #19177C } /* Name.Variable.Class */ -.codehilite .vg { color: #19177C } /* Name.Variable.Global */ -.codehilite .vi { color: #19177C } /* Name.Variable.Instance */ -.codehilite .vm { color: #19177C } /* Name.Variable.Magic */ -.codehilite .il { color: #666666 } /* Literal.Number.Integer.Long */ diff --git a/docs/dynamic-values.md b/docs/dynamic-values.md new file mode 100644 index 000000000..2f5114435 --- /dev/null +++ b/docs/dynamic-values.md @@ -0,0 +1,334 @@ +# Dynamic Values + +Dynamic Values are runtime-evaluated parameters that provide flexible, context-aware configurations for your CloudFormation templates. These values are evaluated during taskcat execution and can pull data from your AWS environment, generate random values, or provide contextual information about your test run. + +## Overview + +Dynamic Values solve common testing challenges: + +- **Environment-specific values** - Pull actual values from your AWS environment +- **Unique resource names** - Generate random strings to avoid naming conflicts +- **Context awareness** - Access current region, project name, and test information +- **Security** - Generate secure passwords and retrieve secrets safely +- **Flexibility** - Reference other parameters and create complex configurations + +## Syntax + +Dynamic Values use the syntax: `$[taskcat_function_name]` or `$[taskcat_function_name_parameter]` + +```yaml +parameters: + DatabasePassword: $[taskcat_genpass_16S] + S3BucketName: $[taskcat_autobucket] + CurrentRegion: $[taskcat_current_region] + AvailabilityZones: $[taskcat_genaz_2] +``` + +## Complete Dynamic Values Reference + +### Random Value Generation + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_random-string]` | Generate 20-character random string | `kj8s9dkf7h3m2n4p5q6r` | Unique resource identifiers | +| `$[taskcat_random-numbers]` | Generate 20-digit random number | `12345678901234567890` | Unique numeric identifiers | +| `$[taskcat_genuuid]` | Generate UUID v1 | `550e8400-e29b-41d4-a716-446655440000` | Globally unique identifiers | + +### Password Generation + +| Dynamic Value | Description | Example | Use Case | +|---------------|-------------|---------|----------| +| `$[taskcat_genpass_8]` | 8-character alphanumeric password | `aB3dE7gH` | Simple passwords | +| `$[taskcat_genpass_16S]` | 16-character password with special chars | `aB3!dE7@gH9#kL2$` | Secure passwords | +| `$[taskcat_genpass_32A]` | 32-character alphanumeric password | `aB3dE7gH9kL2mN4pQ6rS8tU0vW2xY4zA` | Long secure passwords | + +**Password Types:** +- No suffix: Alphanumeric only +- `S`: Includes special characters (!@#$%^&*) +- `A`: Alphanumeric only (explicit) + +### AWS Environment Values + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_current_region]` | Current AWS region | `us-east-1` | Region-specific configurations | +| `$[taskcat_genaz_2]` | 2 availability zones | `us-east-1a,us-east-1b` | Multi-AZ deployments | +| `$[taskcat_genaz_3]` | 3 availability zones | `us-east-1a,us-east-1b,us-east-1c` | High availability setups | +| `$[taskcat_gensingleaz_1]` | Single AZ (1st available) | `us-east-1a` | Single AZ deployments | +| `$[taskcat_gensingleaz_2]` | Single AZ (2nd available) | `us-east-1b` | Specific AZ selection | + +### S3 and Storage + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_autobucket]` | Auto-generated S3 bucket name | `tcat-myproject-us-east-1-123456789` | Template artifacts | +| `$[taskcat_autobucket_prefix]` | S3 bucket prefix | `myproject-us-east-1-123456789` | Custom bucket naming | + +### Context Information + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_project_name]` | Current project name | `my-cloudformation-project` | Tagging and naming | +| `$[taskcat_test_name]` | Current test name | `production-test` | Test identification | +| `$[taskcat_git_branch]` | Current Git branch | `feature/new-feature` | Branch-specific configs | + +### Parameter References + +| Dynamic Value | Description | Example | Use Case | +|---------------|-------------|---------|----------| +| `$[taskcat_getval_ParameterName]` | Reference another parameter | `$[taskcat_getval_DatabasePassword]` | Parameter dependencies | + +### AWS Services Integration + +| Dynamic Value | Description | Example | Use Case | +|---------------|-------------|---------|----------| +| `$[taskcat_ssm_/path/to/parameter]` | Retrieve SSM Parameter | `$[taskcat_ssm_/app/database/host]` | Configuration management | +| `$[taskcat_secretsmanager_secret-name]` | Retrieve Secrets Manager value | `$[taskcat_secretsmanager_prod/db/password]` | Secure credential retrieval | + +### Legacy/Specialized Values + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_getkeypair]` | Default key pair name | `cikey` | EC2 key pair reference | +| `$[taskcat_getlicensebucket]` | License bucket placeholder | `override_this` | License content storage | +| `$[taskcat_getmediabucket]` | Media bucket placeholder | `override_this` | Media content storage | + +## Advanced Examples + +### Multi-Tier Application + +```yaml +project: + name: multi-tier-app + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + +global: + parameters: + ProjectName: $[taskcat_project_name] + Environment: production + +tests: + vpc-infrastructure: + template: templates/vpc.yaml + parameters: + VpcName: $[taskcat_project_name]-vpc-$[taskcat_current_region] + AvailabilityZones: $[taskcat_genaz_3] + + database-tier: + template: templates/rds.yaml + parameters: + DBInstanceIdentifier: $[taskcat_project_name]-db-$[taskcat_random-string] + MasterUsername: admin + MasterUserPassword: $[taskcat_genpass_32S] + DBSubnetGroupName: $[taskcat_getval_VpcName]-db-subnets + + application-tier: + template: templates/app.yaml + parameters: + ApplicationName: $[taskcat_project_name]-app + InstanceType: m5.large + KeyName: $[taskcat_getkeypair] + S3Bucket: $[taskcat_autobucket] + DatabaseEndpoint: $[taskcat_getval_DBInstanceIdentifier] + + monitoring: + template: templates/monitoring.yaml + parameters: + DashboardName: $[taskcat_project_name]-$[taskcat_test_name]-dashboard + LogGroupName: /aws/lambda/$[taskcat_project_name] + AlertEmail: $[taskcat_ssm_/notifications/email] +``` + +### Environment-Specific Configuration + +```yaml +project: + name: environment-configs + regions: + - us-east-1 + +tests: + development: + template: templates/app.yaml + parameters: + Environment: dev + InstanceType: t3.micro + DatabasePassword: $[taskcat_genpass_16] + S3Bucket: $[taskcat_project_name]-dev-$[taskcat_current_region] + + staging: + template: templates/app.yaml + parameters: + Environment: staging + InstanceType: t3.small + DatabasePassword: $[taskcat_secretsmanager_staging/db/password] + S3Bucket: $[taskcat_project_name]-staging-$[taskcat_current_region] + + production: + template: templates/app.yaml + parameters: + Environment: prod + InstanceType: m5.large + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + S3Bucket: $[taskcat_project_name]-prod-$[taskcat_current_region] + BackupRetention: 30 + MonitoringEnabled: true +``` + +### Security-Focused Configuration + +```yaml +project: + name: secure-app + regions: + - us-east-1 + - us-west-2 + +tests: + secure-deployment: + template: templates/secure-app.yaml + parameters: + # Generate unique, secure passwords + DatabaseMasterPassword: $[taskcat_genpass_32S] + ApplicationSecret: $[taskcat_genpass_24S] + + # Use AWS Secrets Manager for production secrets + ApiKey: $[taskcat_secretsmanager_prod/api/key] + CertificateArn: $[taskcat_ssm_/ssl/certificate/arn] + + # Generate unique resource names + KMSKeyAlias: $[taskcat_project_name]-key-$[taskcat_genuuid] + S3BucketName: $[taskcat_autobucket] + + # Context-aware naming + LogGroupName: /aws/lambda/$[taskcat_project_name]-$[taskcat_current_region] + + # Reference other parameters + DatabasePasswordConfirm: $[taskcat_getval_DatabaseMasterPassword] +``` + +### Multi-Region Deployment + +```yaml +project: + name: global-app + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 + +tests: + global-infrastructure: + template: templates/global-app.yaml + parameters: + # Region-specific configurations + PrimaryRegion: us-east-1 + CurrentRegion: $[taskcat_current_region] + + # Generate region-specific AZs + AvailabilityZones: $[taskcat_genaz_2] + + # Unique naming per region + S3BucketName: $[taskcat_project_name]-$[taskcat_current_region]-$[taskcat_random-numbers] + + # Global unique identifiers + DeploymentId: $[taskcat_genuuid] + + # Branch-specific configurations + GitBranch: $[taskcat_git_branch] + + # Environment from SSM + Environment: $[taskcat_ssm_/global/environment] +``` + +## Best Practices + +### 1. Use Appropriate Value Types + +```yaml +# ✅ Good: Use specific types for specific purposes +parameters: + DatabasePassword: $[taskcat_genpass_16S] # Secure password + ResourceId: $[taskcat_genuuid] # Globally unique + BucketName: $[taskcat_autobucket] # S3-compliant naming + +# ❌ Avoid: Using generic values for specific purposes +parameters: + DatabasePassword: $[taskcat_random-string] # Not secure enough + ResourceId: $[taskcat_random-numbers] # May not be unique +``` + +### 2. Leverage Parameter References + +```yaml +# ✅ Good: Reference parameters to maintain consistency +parameters: + MasterPassword: $[taskcat_genpass_20S] + PasswordConfirm: $[taskcat_getval_MasterPassword] + +# ❌ Avoid: Generating separate values for related parameters +parameters: + MasterPassword: $[taskcat_genpass_20S] + PasswordConfirm: $[taskcat_genpass_20S] # Different values! +``` + +### 3. Use Context-Aware Naming + +```yaml +# ✅ Good: Include context in resource names +parameters: + LogGroup: /aws/lambda/$[taskcat_project_name]-$[taskcat_current_region] + S3Bucket: $[taskcat_project_name]-logs-$[taskcat_current_region] + +# ❌ Avoid: Generic naming that may conflict +parameters: + LogGroup: /aws/lambda/myapp + S3Bucket: myapp-logs +``` + +### 4. Secure Credential Management + +```yaml +# ✅ Good: Use AWS services for production secrets +parameters: + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + ApiKey: $[taskcat_ssm_/app/api/key] + +# ✅ Good: Generate secure passwords for testing +parameters: + TestPassword: $[taskcat_genpass_16S] + +# ❌ Avoid: Hardcoded secrets +parameters: + DatabasePassword: "hardcoded-password" +``` + +## Troubleshooting + +### Common Issues + +**Dynamic Value not replaced:** +- Check syntax: `$[taskcat_function_name]` +- Verify function name spelling +- Ensure proper parameter placement + +**AWS service integration fails:** +- Verify IAM permissions for SSM/Secrets Manager +- Check parameter/secret exists in target region +- Validate parameter path format + +**AZ generation fails:** +- Check if region has enough AZs +- Verify region is enabled in your account +- Consider AZ exclusions in configuration + +**Parameter reference fails:** +- Ensure referenced parameter exists +- Check parameter name spelling +- Verify parameter is defined before reference + +For more troubleshooting help, see the [Troubleshooting Guide](troubleshooting.md). diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 000000000..352a9890c --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,653 @@ +# Examples + +Explore real-world taskcat configurations and learn from practical implementations. These examples demonstrate best practices, advanced features, and common use cases. + +## Basic Examples + +### Simple S3 Bucket Test + +```yaml +# .taskcat.yml +project: + name: simple-s3-test + regions: + - us-east-1 + +tests: + basic: + template: s3-bucket.yaml + parameters: + BucketName: $[taskcat_autobucket] +``` + +```yaml +# s3-bucket.yaml +AWSTemplateFormatVersion: '2010-09-09' +Description: Simple S3 bucket + +Parameters: + BucketName: + Type: String + +Resources: + TestBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref BucketName + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + +Outputs: + BucketName: + Value: !Ref TestBucket + BucketArn: + Value: !GetAtt TestBucket.Arn +``` + +### Multi-Region VPC Test + +```yaml +# .taskcat.yml +project: + name: vpc-multi-region + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + +tests: + vpc-test: + template: vpc.yaml + parameters: + VpcCidr: 10.0.0.0/16 + AvailabilityZones: $[taskcat_genaz_2] + Environment: test +``` + +## Intermediate Examples + +### Web Application with Database + +```yaml +# .taskcat.yml +project: + name: web-app-with-db + regions: + - us-east-1 + - us-west-2 + parameters: + ProjectName: web-application + Environment: staging + +tests: + vpc-infrastructure: + template: templates/vpc.yaml + parameters: + VpcCidr: 10.0.0.0/16 + AvailabilityZones: $[taskcat_genaz_3] + + database: + template: templates/rds.yaml + parameters: + DatabaseName: webapp + DatabaseUsername: admin + DatabasePassword: $[taskcat_genpass_16S] + DatabaseInstanceClass: db.t3.micro + + web-application: + template: templates/web-app.yaml + parameters: + InstanceType: t3.medium + MinSize: 2 + MaxSize: 6 + KeyName: $[taskcat_getkeypair] + S3Bucket: $[taskcat_autobucket] +``` + +### Serverless Application + +```yaml +# .taskcat.yml +project: + name: serverless-api + regions: + - us-east-1 + - us-west-2 + lambda_source_path: src/functions + lambda_zip_path: dist/functions + package_lambda: true + +tests: + api-gateway: + template: templates/api-gateway.yaml + parameters: + ApiName: $[taskcat_project_name]-api + StageName: $[taskcat_test_name] + + lambda-functions: + template: templates/lambda.yaml + parameters: + FunctionName: $[taskcat_project_name]-function + Runtime: python3.9 + MemorySize: 256 + Timeout: 30 + S3Bucket: $[taskcat_autobucket] + + dynamodb-table: + template: templates/dynamodb.yaml + parameters: + TableName: $[taskcat_project_name]-table + BillingMode: PAY_PER_REQUEST +``` + +## Advanced Examples + +### Multi-Environment Enterprise Application + +```yaml +# .taskcat.yml +project: + name: enterprise-app + owner: platform-team@company.com + s3_regional_buckets: true + package_lambda: true + +general: + parameters: + ProjectName: enterprise-application + Owner: platform-team + tags: + CostCenter: "1001" + Department: Engineering + +tests: + development: + template: templates/main.yaml + regions: + - us-east-1 + parameters: + Environment: dev + InstanceType: t3.micro + DatabaseInstanceClass: db.t3.micro + MinSize: 1 + MaxSize: 2 + EnableMonitoring: false + BackupRetentionPeriod: 1 + + staging: + template: templates/main.yaml + regions: + - us-east-1 + - us-west-2 + parameters: + Environment: staging + InstanceType: t3.medium + DatabaseInstanceClass: db.t3.small + MinSize: 2 + MaxSize: 4 + EnableMonitoring: true + BackupRetentionPeriod: 7 + DatabasePassword: $[taskcat_genpass_20S] + + production: + template: templates/main.yaml + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + parameters: + Environment: prod + InstanceType: m5.large + DatabaseInstanceClass: db.r5.large + MinSize: 3 + MaxSize: 10 + EnableMonitoring: true + EnableEncryption: true + MultiAZ: true + BackupRetentionPeriod: 30 + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + SSLCertificateArn: $[taskcat_ssm_/ssl/certificate/arn] + auth: + us-east-1: production-profile + us-west-2: production-profile + eu-west-1: europe-profile +``` + +### Cross-Account Deployment + +```yaml +# .taskcat.yml +project: + name: cross-account-app + +tests: + shared-services-account: + template: templates/shared-services.yaml + regions: + - us-east-1 + auth: + default: shared-services-profile + parameters: + Environment: shared + VpcCidr: 10.0.0.0/16 + + development-account: + template: templates/application.yaml + regions: + - us-east-1 + auth: + default: dev-account-profile + parameters: + Environment: dev + SharedServicesVpcId: $[taskcat_ssm_/shared/vpc/id] + InstanceType: t3.micro + + production-account: + template: templates/application.yaml + regions: + - us-east-1 + - us-west-2 + auth: + default: prod-account-profile + parameters: + Environment: prod + SharedServicesVpcId: $[taskcat_ssm_/shared/vpc/id] + InstanceType: m5.large + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] +``` + +### Microservices Architecture + +```yaml +# .taskcat.yml +project: + name: microservices-platform + regions: + - us-east-1 + - us-west-2 + lambda_source_path: services + package_lambda: true + +general: + parameters: + Platform: microservices + Environment: production + tags: + Architecture: microservices + ManagedBy: taskcat + +tests: + infrastructure: + template: templates/infrastructure.yaml + parameters: + VpcCidr: 10.0.0.0/16 + AvailabilityZones: $[taskcat_genaz_3] + ClusterName: $[taskcat_project_name]-cluster + + user-service: + template: templates/microservice.yaml + parameters: + ServiceName: user-service + ContainerImage: user-service:latest + ContainerPort: 8080 + DesiredCount: 3 + DatabaseName: users + DatabasePassword: $[taskcat_genpass_20S] + + order-service: + template: templates/microservice.yaml + parameters: + ServiceName: order-service + ContainerImage: order-service:latest + ContainerPort: 8081 + DesiredCount: 2 + DatabaseName: orders + DatabasePassword: $[taskcat_genpass_20S] + + notification-service: + template: templates/lambda-service.yaml + parameters: + ServiceName: notification-service + Runtime: python3.9 + MemorySize: 512 + Timeout: 60 + QueueName: $[taskcat_project_name]-notifications + + api-gateway: + template: templates/api-gateway.yaml + parameters: + ApiName: $[taskcat_project_name]-api + StageName: v1 + UserServiceEndpoint: $[taskcat_getval_UserServiceEndpoint] + OrderServiceEndpoint: $[taskcat_getval_OrderServiceEndpoint] +``` + +### Data Pipeline + +```yaml +# .taskcat.yml +project: + name: data-pipeline + regions: + - us-east-1 + parameters: + DataBucket: $[taskcat_autobucket] + Environment: production + +tests: + data-ingestion: + template: templates/data-ingestion.yaml + parameters: + KinesisStreamName: $[taskcat_project_name]-stream + KinesisShardCount: 2 + FirehoseDeliveryStreamName: $[taskcat_project_name]-firehose + S3Bucket: $[taskcat_getval_DataBucket] + + data-processing: + template: templates/data-processing.yaml + parameters: + GlueJobName: $[taskcat_project_name]-etl + GlueJobScript: s3://$[taskcat_getval_DataBucket]/scripts/etl.py + DatabaseName: $[taskcat_project_name]_db + TableName: processed_data + + data-analytics: + template: templates/data-analytics.yaml + parameters: + RedshiftClusterIdentifier: $[taskcat_project_name]-cluster + RedshiftDatabaseName: analytics + RedshiftMasterUsername: admin + RedshiftMasterPassword: $[taskcat_genpass_20S] + RedshiftNodeType: dc2.large + RedshiftNumberOfNodes: 2 +``` + +## Specialized Examples + +### Security-Focused Deployment + +```yaml +# .taskcat.yml +project: + name: secure-application + regions: + - us-east-1 + - us-west-2 + +tests: + security-baseline: + template: templates/security-baseline.yaml + parameters: + EnableCloudTrail: true + EnableGuardDuty: true + EnableSecurityHub: true + EnableConfig: true + CloudTrailS3Bucket: $[taskcat_autobucket] + + encrypted-application: + template: templates/encrypted-app.yaml + parameters: + KMSKeyAlias: $[taskcat_project_name]-key + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + SSLCertificateArn: $[taskcat_ssm_/ssl/certificate/arn] + EnableEncryptionAtRest: true + EnableEncryptionInTransit: true + InstanceType: m5.large + + compliance-monitoring: + template: templates/compliance.yaml + parameters: + ComplianceFramework: SOC2 + LogRetentionPeriod: 2557 # 7 years in days + EnableLogEncryption: true + MonitoringS3Bucket: $[taskcat_autobucket] +``` + +### Disaster Recovery Setup + +```yaml +# .taskcat.yml +project: + name: disaster-recovery + +tests: + primary-region: + template: templates/primary-infrastructure.yaml + regions: + - us-east-1 + parameters: + Environment: production + IsPrimaryRegion: true + DatabaseInstanceClass: db.r5.xlarge + MultiAZ: true + BackupRetentionPeriod: 35 + CrossRegionBackupEnabled: true + ReplicationTargetRegion: us-west-2 + + disaster-recovery-region: + template: templates/dr-infrastructure.yaml + regions: + - us-west-2 + parameters: + Environment: production + IsDRRegion: true + DatabaseInstanceClass: db.r5.large + ReadReplicaSourceRegion: us-east-1 + AutomatedBackupRetentionPeriod: 35 + + failover-automation: + template: templates/failover-automation.yaml + regions: + - us-east-1 + - us-west-2 + parameters: + PrimaryRegion: us-east-1 + DRRegion: us-west-2 + Route53HealthCheckUrl: https://api.example.com/health + FailoverThreshold: 3 +``` + +## Testing Patterns + +### Blue-Green Deployment Testing + +```yaml +# .taskcat.yml +project: + name: blue-green-deployment + +tests: + blue-environment: + template: templates/application.yaml + parameters: + Environment: blue + Version: v1.0.0 + TrafficWeight: 100 + InstanceType: m5.large + + green-environment: + template: templates/application.yaml + parameters: + Environment: green + Version: v1.1.0 + TrafficWeight: 0 + InstanceType: m5.large + + traffic-shifting: + template: templates/traffic-manager.yaml + parameters: + BlueEnvironmentArn: $[taskcat_getval_BlueEnvironmentArn] + GreenEnvironmentArn: $[taskcat_getval_GreenEnvironmentArn] + InitialTrafficPercentage: 10 +``` + +### A/B Testing Infrastructure + +```yaml +# .taskcat.yml +project: + name: ab-testing-platform + +tests: + variant-a: + template: templates/application-variant.yaml + parameters: + VariantName: A + FeatureFlags: feature-a-enabled + InstanceType: m5.large + TrafficPercentage: 50 + + variant-b: + template: templates/application-variant.yaml + parameters: + VariantName: B + FeatureFlags: feature-b-enabled + InstanceType: m5.large + TrafficPercentage: 50 + + analytics-infrastructure: + template: templates/analytics.yaml + parameters: + KinesisStreamName: $[taskcat_project_name]-events + ElasticsearchDomain: $[taskcat_project_name]-analytics + KibanaDashboardName: ab-testing-dashboard +``` + +## Best Practices Examples + +### Parameterized and Reusable + +```yaml +# .taskcat.yml - Good example of reusable configuration +project: + name: reusable-infrastructure + parameters: + # Common parameters + ProjectName: $[taskcat_project_name] + Owner: platform-team + +general: + parameters: + # Global defaults + Environment: test + EnableMonitoring: true + tags: + ManagedBy: taskcat + Project: $[taskcat_project_name] + +tests: + # Small deployment for development + small: + template: templates/scalable-app.yaml + parameters: + Size: small + InstanceType: t3.micro + MinSize: 1 + MaxSize: 2 + DatabaseInstanceClass: db.t3.micro + + # Medium deployment for staging + medium: + template: templates/scalable-app.yaml + parameters: + Size: medium + InstanceType: t3.medium + MinSize: 2 + MaxSize: 4 + DatabaseInstanceClass: db.t3.small + + # Large deployment for production + large: + template: templates/scalable-app.yaml + parameters: + Size: large + InstanceType: m5.large + MinSize: 3 + MaxSize: 10 + DatabaseInstanceClass: db.r5.large + EnableEncryption: true + MultiAZ: true +``` + +### Security Best Practices + +```yaml +# .taskcat.yml - Security-focused configuration +project: + name: secure-by-design + +tests: + security-compliant: + template: templates/secure-application.yaml + parameters: + # Use Secrets Manager for sensitive data + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + ApiKey: $[taskcat_secretsmanager_prod/api/key] + + # Use SSM for configuration + DatabaseEndpoint: $[taskcat_ssm_/app/database/endpoint] + CacheEndpoint: $[taskcat_ssm_/app/cache/endpoint] + + # Generate unique, secure values + EncryptionKey: $[taskcat_genuuid] + S3Bucket: $[taskcat_autobucket] + + # Security settings + EnableEncryption: true + EnableLogging: true + EnableMonitoring: true + RestrictPublicAccess: true +``` + +## Integration Examples + +### CI/CD Pipeline Integration + +```yaml +# .taskcat.yml for CI/CD +project: + name: cicd-integration + +tests: + pull-request: + template: templates/app.yaml + regions: + - us-east-1 + parameters: + Environment: pr-$[taskcat_git_branch] + InstanceType: t3.micro + + staging-deployment: + template: templates/app.yaml + regions: + - us-east-1 + - us-west-2 + parameters: + Environment: staging + InstanceType: t3.medium + GitCommit: $[taskcat_git_branch] + + production-deployment: + template: templates/app.yaml + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + parameters: + Environment: production + InstanceType: m5.large + GitCommit: $[taskcat_git_branch] + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] +``` + +These examples demonstrate the flexibility and power of taskcat for testing CloudFormation templates across various scenarios, from simple single-resource tests to complex multi-tier applications and enterprise deployments. + +## Next Steps + +- [Configuration Guide](configuration.md) - Detailed configuration options +- [Dynamic Values](dynamic-values.md) - Runtime-evaluated parameters +- [Parameter Overrides](parameter-overrides.md) - Advanced parameter techniques +- [Schema Reference](schema.md) - Complete configuration reference diff --git a/docs/examples/advanced.md b/docs/examples/advanced.md new file mode 100644 index 000000000..702e3fd96 --- /dev/null +++ b/docs/examples/advanced.md @@ -0,0 +1,86 @@ +# Advanced Examples + +This page covers more complex taskcat configurations and use cases. + +## Multi-Test Configuration + +```yaml +# .taskcat.yml +project: + name: enterprise-app + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + +global: + parameters: + Environment: testing + Owner: devops-team + +tests: + infrastructure: + template: templates/infrastructure.yaml + parameters: + VpcCidr: 10.0.0.0/16 + AvailabilityZones: $[taskcat_genaz_3] + + application: + template: templates/application.yaml + parameters: + InstanceType: t3.medium + DatabasePassword: $[taskcat_genpass_16S] + + monitoring: + template: templates/monitoring.yaml + regions: + - us-east-1 # Only deploy monitoring in primary region +``` + +## Using AWS Service Integration + +```yaml +tests: + app-with-secrets: + template: templates/app.yaml + parameters: + # Get AMI ID from SSM Parameter Store + LatestAMI: $[taskcat_ssm_/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2] + + # Get database credentials from Secrets Manager + DatabaseCredentials: $[taskcat_secretsmanager_prod/database/master] + + # Use current region for region-specific resources + DeploymentRegion: $[taskcat_current_region] + + # Generate unique identifiers + UniqueId: $[taskcat_genuuid] +``` + +## Parameter Validation + +```yaml +tests: + parameter-validation: + template: templates/app.yaml + parameters: + # Password with confirmation + AdminPassword: $[taskcat_genpass_12S] + ConfirmPassword: $[taskcat_getval_AdminPassword] + + # Consistent naming + ProjectName: $[taskcat_project_name] + TestName: $[taskcat_test_name] +``` + +## Custom Authentication + +```yaml +project: + name: multi-account-test + auth: + us-east-1: production-profile + us-west-2: staging-profile + eu-west-1: development-profile + default: default-profile +``` diff --git a/docs/examples/basic.md b/docs/examples/basic.md new file mode 100644 index 000000000..d6b8771f2 --- /dev/null +++ b/docs/examples/basic.md @@ -0,0 +1,73 @@ +# Basic Usage Examples + +This page provides simple, practical examples to help you get started with taskcat. + +## Simple S3 Bucket Test + +```yaml +# .taskcat.yml +project: + name: simple-s3-test + regions: + - us-east-1 + - us-west-2 + +tests: + s3-bucket: + template: templates/s3-bucket.yaml + parameters: + BucketName: $[taskcat_autobucket] +``` + +```yaml +# templates/s3-bucket.yaml +AWSTemplateFormatVersion: '2010-09-09' +Description: Simple S3 bucket + +Parameters: + BucketName: + Type: String + Description: Name of the S3 bucket + +Resources: + TestBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref BucketName + +Outputs: + BucketName: + Value: !Ref TestBucket + Description: Name of the created bucket +``` + +## Multi-AZ VPC Test + +```yaml +# .taskcat.yml +project: + name: vpc-test + regions: + - us-east-1 + - eu-west-1 + +tests: + vpc-multi-az: + template: templates/vpc.yaml + parameters: + AvailabilityZones: $[taskcat_genaz_2] + VpcCidr: 10.0.0.0/16 +``` + +## Running the Tests + +```bash +# Run all tests +taskcat test run + +# Run specific test +taskcat test run --test-names s3-bucket + +# Run with custom output directory +taskcat test run --output-directory ./my-results +``` diff --git a/docs/examples/index.md b/docs/examples/index.md new file mode 100644 index 000000000..90fcdbb8a --- /dev/null +++ b/docs/examples/index.md @@ -0,0 +1,116 @@ +# Examples + +Explore real-world taskcat configurations and learn from practical implementations. These examples demonstrate best practices, advanced features, and common use cases. + +## Quick Reference + +
+
+

🚀 Basic Usage

+

Simple configurations to get you started with taskcat testing.

+ View Examples +
+ +
+

⚡ Advanced Scenarios

+

Complex multi-tier applications and enterprise-grade configurations.

+ Explore Advanced +
+
+ +## Featured Examples + +### Multi-Region Web Application + +```yaml +project: + name: web-application + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + +global: + parameters: + ProjectName: $[taskcat_project_name] + Environment: production + +tests: + vpc-infrastructure: + template: templates/vpc.yaml + parameters: + VpcName: $[taskcat_project_name]-vpc-$[taskcat_current_region] + AvailabilityZones: $[taskcat_genaz_3] + + web-tier: + template: templates/web-tier.yaml + parameters: + ApplicationName: $[taskcat_project_name]-web + InstanceType: t3.medium + S3Bucket: $[taskcat_autobucket] + SSLCertificate: $[taskcat_ssm_/ssl/certificate/arn] +``` + +### Serverless Application + +```yaml +project: + name: serverless-api + regions: + - us-east-1 + - us-west-2 + +tests: + api-gateway: + template: templates/api-gateway.yaml + parameters: + ApiName: $[taskcat_project_name]-api-$[taskcat_current_region] + StageName: $[taskcat_test_name] + + lambda-functions: + template: templates/lambda.yaml + parameters: + FunctionName: $[taskcat_project_name]-function + Runtime: python3.9 + S3Bucket: $[taskcat_autobucket] + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] +``` + +### Database Cluster + +```yaml +project: + name: database-cluster + regions: + - us-east-1 + - us-west-2 + +tests: + aurora-cluster: + template: templates/aurora.yaml + parameters: + ClusterIdentifier: $[taskcat_project_name]-cluster-$[taskcat_genuuid] + MasterUsername: admin + MasterUserPassword: $[taskcat_genpass_32S] + DatabaseName: $[taskcat_project_name] + BackupRetentionPeriod: 7 + PreferredBackupWindow: "03:00-04:00" + PreferredMaintenanceWindow: "sun:04:00-sun:05:00" +``` + +## Browse All Examples + +- **[Basic Usage](basic.md)** - Simple, straightforward examples +- **[Advanced Scenarios](advanced.md)** - Complex, production-ready configurations + +## Contributing Examples + +Have a great taskcat configuration to share? We'd love to include it! Examples should: + +- ✅ Follow taskcat best practices +- ✅ Include clear documentation +- ✅ Demonstrate real-world use cases +- ✅ Use Dynamic Values appropriately +- ✅ Be production-ready + +Submit your examples via [GitHub Issues](https://github.com/aws-ia/taskcat/issues) or [Pull Requests](https://github.com/aws-ia/taskcat/pulls). diff --git a/docs/gen_ref_pages.py b/docs/gen_ref_pages.py new file mode 100644 index 000000000..33bc0ecb3 --- /dev/null +++ b/docs/gen_ref_pages.py @@ -0,0 +1,32 @@ +"""Generate the code reference pages and navigation.""" + +from pathlib import Path + +import mkdocs_gen_files + +nav = mkdocs_gen_files.Nav() + +for path in sorted(Path("taskcat").rglob("*.py")): + module_path = path.relative_to(".").with_suffix("") + doc_path = path.relative_to(".").with_suffix(".md") + full_doc_path = Path("reference", doc_path) + + parts = tuple(module_path.parts) + + if parts[-1] == "__init__": + parts = parts[:-1] + doc_path = doc_path.with_name("index.md") + full_doc_path = full_doc_path.with_name("index.md") + elif parts[-1] == "__main__": + continue + + nav[parts] = doc_path.as_posix() + + with mkdocs_gen_files.open(full_doc_path, "w") as fd: + ident = ".".join(parts) + fd.write(f"::: {ident}") + + mkdocs_gen_files.set_edit_path(full_doc_path, path) + +with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file: + nav_file.writelines(nav.build_literate_nav()) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md new file mode 100644 index 000000000..d40482f49 --- /dev/null +++ b/docs/getting-started/configuration.md @@ -0,0 +1,357 @@ +# Configuration Guide + +Learn how to configure taskcat for your specific testing needs with comprehensive configuration options. + +## Configuration File Structure + +taskcat uses YAML configuration files (`.taskcat.yml`) with this structure: + +```yaml +project: + name: string # Project name + regions: [list] # AWS regions to test + s3_bucket: string # Optional: Custom S3 bucket + s3_key_prefix: string # Optional: S3 key prefix + +tests: + test-name: # Test identifier + template: string # Path to CloudFormation template + parameters: {} # Parameter overrides + regions: [list] # Optional: Test-specific regions + +global: + parameters: {} # Global parameter overrides +``` + +## Project Configuration + +### Basic Project Settings + +```yaml +project: + name: my-cloudformation-project + regions: + - us-east-1 + - us-west-2 + - eu-west-1 +``` + +### Advanced Project Settings + +```yaml +project: + name: enterprise-infrastructure + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 + s3_bucket: my-custom-taskcat-bucket + s3_key_prefix: testing/templates/ + tags: + Environment: Testing + Project: taskcat + Owner: DevOps-Team +``` + +## Test Configuration + +### Single Test + +```yaml +tests: + basic-test: + template: templates/main.yaml + parameters: + InstanceType: t3.micro + Environment: test +``` + +### Multiple Tests + +```yaml +tests: + small-deployment: + template: templates/small.yaml + parameters: + InstanceType: t3.micro + + large-deployment: + template: templates/large.yaml + parameters: + InstanceType: m5.xlarge + + multi-az-test: + template: templates/multi-az.yaml + regions: + - us-east-1 + - us-west-2 + parameters: + AvailabilityZones: $[taskcat_genaz_3] +``` + +## Parameter Management + +### Global Parameters + +Parameters that apply to all tests: + +```yaml +global: + parameters: + KeyPairName: my-keypair + VpcCidr: 10.0.0.0/16 + Environment: testing + +tests: + test1: + template: templates/app.yaml + # Inherits global parameters + test2: + template: templates/db.yaml + # Also inherits global parameters +``` + +### Test-Specific Parameters + +Override global parameters for specific tests: + +```yaml +global: + parameters: + Environment: testing + InstanceType: t3.micro + +tests: + production-test: + template: templates/app.yaml + parameters: + Environment: production # Overrides global + InstanceType: m5.large # Overrides global +``` + +### Pseudo-Parameters + +Use dynamic parameters for flexible testing: + +```yaml +tests: + dynamic-test: + template: templates/app.yaml + parameters: + # Generate random values + DatabasePassword: $[taskcat_genpass_16S] + S3Bucket: $[taskcat_autobucket] + + # Use current context + Region: $[taskcat_current_region] + ProjectName: $[taskcat_project_name] + + # Generate availability zones + AvailabilityZones: $[taskcat_genaz_2] + + # Reference other parameters + PasswordConfirm: $[taskcat_getval_DatabasePassword] +``` + +## Region Configuration + +### Project-Level Regions + +All tests use these regions by default: + +```yaml +project: + regions: + - us-east-1 + - us-west-2 + - eu-west-1 +``` + +### Test-Specific Regions + +Override regions for specific tests: + +```yaml +project: + regions: + - us-east-1 + - us-west-2 + +tests: + global-test: + template: templates/global.yaml + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 + + us-only-test: + template: templates/us-specific.yaml + regions: + - us-east-1 + - us-west-2 +``` + +## Advanced Configuration + +### Custom S3 Configuration + +```yaml +project: + name: my-project + s3_bucket: my-custom-bucket-${AWS::Region} + s3_key_prefix: taskcat-tests/ + s3_object_acl: private +``` + +### Authentication Configuration + +```yaml +project: + auth: + us-east-1: profile1 + us-west-2: profile2 + default: default-profile +``` + +### Template Processing + +```yaml +project: + template: + transforms: + - AWS::Serverless-2016-10-31 + capabilities: + - CAPABILITY_IAM + - CAPABILITY_NAMED_IAM +``` + +## Configuration Examples + +### Microservices Architecture + +```yaml +project: + name: microservices-platform + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + +global: + parameters: + Environment: testing + VpcCidr: 10.0.0.0/16 + +tests: + vpc-infrastructure: + template: templates/vpc.yaml + + application-tier: + template: templates/app-tier.yaml + parameters: + InstanceType: t3.medium + + database-tier: + template: templates/db-tier.yaml + parameters: + DBInstanceClass: db.t3.micro + + monitoring: + template: templates/monitoring.yaml + regions: + - us-east-1 # Only deploy monitoring in primary region +``` + +### Multi-Environment Testing + +```yaml +project: + name: multi-env-app + regions: + - us-east-1 + - us-west-2 + +tests: + development: + template: templates/app.yaml + parameters: + Environment: dev + InstanceType: t3.micro + + staging: + template: templates/app.yaml + parameters: + Environment: staging + InstanceType: t3.small + + production: + template: templates/app.yaml + parameters: + Environment: prod + InstanceType: m5.large +``` + +## Best Practices + +### 1. Use Meaningful Names +```yaml +tests: + vpc-with-public-subnets: # ✅ Descriptive + template: templates/vpc.yaml + + test1: # ❌ Not descriptive + template: templates/vpc.yaml +``` + +### 2. Organize Parameters +```yaml +global: + parameters: + # Common across all tests + Environment: testing + Owner: devops-team + +tests: + web-tier: + parameters: + # Specific to this test + InstanceType: t3.medium + MinSize: 2 + MaxSize: 10 +``` + +### 3. Use Pseudo-Parameters +```yaml +parameters: + # ✅ Dynamic and flexible + DatabasePassword: $[taskcat_genpass_16S] + S3Bucket: $[taskcat_autobucket] + + # ❌ Static and potentially conflicting + DatabasePassword: hardcoded-password + S3Bucket: my-static-bucket-name +``` + +## Validation + +Validate your configuration: + +```bash +# Check configuration syntax +taskcat test run --dry-run + +# Lint CloudFormation templates +taskcat lint + +# Generate configuration schema +taskcat schema +``` + +## Next Steps + +- [Dynamic Values Guide](../usage/DYNAMIC_VALUES.md) +- [Parameter Overrides](../usage/PARAMETER_OVERRIDES.md) +- [Advanced Examples](../examples/advanced.md) diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md new file mode 100644 index 000000000..297b1584f --- /dev/null +++ b/docs/getting-started/index.md @@ -0,0 +1,92 @@ +# Getting Started with taskcat + +Welcome to taskcat! This section will get you up and running with AWS CloudFormation template testing in minutes. + +## What You'll Learn + +
+
+

📦 Installation

+

Multiple installation methods including pip, Docker, and from source.

+ Install taskcat +
+ +
+

🚀 Quick Start

+

Run your first test in under 5 minutes with our step-by-step guide.

+ Quick Start +
+ +
+

⚙️ Configuration

+

Master taskcat configuration for advanced testing scenarios.

+ Configure +
+
+ +## Learning Path + +Follow this recommended path to master taskcat: + +### 1. **Installation** (5 minutes) +Get taskcat installed on your system with your preferred method. + +### 2. **Quick Start** (10 minutes) +Create and run your first test to understand the basics. + +### 3. **Configuration** (20 minutes) +Learn about advanced configuration options and best practices. + +### 4. **Dynamic Values** (15 minutes) +Master runtime-evaluated parameters for flexible testing. + +## Prerequisites + +Before you begin, ensure you have: + +- **AWS Account** with appropriate permissions +- **Python 3.8+** installed on your system +- **AWS CLI** configured with credentials +- **Basic CloudFormation knowledge** + +## Quick Installation + +```bash +# Install taskcat via pip +pip install taskcat + +# Verify installation +taskcat --version + +# Get help +taskcat --help +``` + +## Your First Test + +```yaml +# .taskcat.yml +project: + name: my-first-test + regions: + - us-east-1 + +tests: + basic: + template: template.yaml + parameters: + BucketName: $[taskcat_autobucket] +``` + +```bash +# Run the test +taskcat test run +``` + +## Need Help? + +- 📚 **[Documentation](../usage/GENERAL_USAGE.md)** - Comprehensive guides +- 💬 **[Community](../support/troubleshooting.md)** - Get help from other users +- 🐛 **[Issues](https://github.com/aws-ia/taskcat/issues)** - Report bugs or request features + +Ready to begin? Start with [Installation](installation.md)! diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 000000000..88973b107 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,126 @@ +# Installation Guide + +Get taskcat installed and running on your system with our comprehensive installation guide. + +## Prerequisites + +Before installing taskcat, ensure you have: + +- **Python 3.8+**: taskcat requires Python 3.8 or higher +- **AWS CLI**: Configured with appropriate credentials +- **Git**: For cloning repositories and version control +- **Sufficient AWS Permissions**: See [Required Permissions](#required-permissions) + +## Installation Methods + +### Method 1: PyPI (Recommended) + +The easiest way to install taskcat is via PyPI: + +```bash +pip install taskcat +``` + +For the latest development version: + +```bash +pip install --upgrade taskcat +``` + +### Method 2: From Source + +For development or the latest features: + +```bash +git clone https://github.com/aws-ia/taskcat.git +cd taskcat +pip install -e . +``` + +### Method 3: Docker + +Use our pre-built Docker images: + +```bash +docker pull public.ecr.aws/aws-ia/taskcat:latest +docker run -it --rm -v $(pwd):/workspace taskcat --help +``` + +## Verification + +Verify your installation: + +```bash +taskcat --version +taskcat --help +``` + +## AWS Configuration + +### Configure AWS Credentials + +taskcat uses AWS credentials from your environment. Configure using: + +#### AWS CLI +```bash +aws configure +``` + +#### Environment Variables +```bash +export AWS_ACCESS_KEY_ID=your-access-key +export AWS_SECRET_ACCESS_KEY=your-secret-key +export AWS_DEFAULT_REGION=us-east-1 +``` + +#### IAM Roles (Recommended for EC2/Lambda) +taskcat automatically uses IAM roles when running on AWS services. + +### Required Permissions + +taskcat requires the following AWS permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "cloudformation:*", + "s3:*", + "iam:*", + "ec2:Describe*", + "ssm:GetParameter*", + "secretsmanager:GetSecretValue" + ], + "Resource": "*" + } + ] +} +``` + +## Next Steps + +Once installed, continue to the [Quick Start Guide](quickstart.md) to run your first test. + +## Troubleshooting + +### Common Issues + +**Python version too old:** +```bash +python3 --version # Should be 3.8+ +``` + +**Permission denied:** +```bash +pip install --user taskcat +``` + +**AWS credentials not found:** +```bash +aws configure list +``` + +For more help, see our [Troubleshooting Guide](../support/troubleshooting.md). diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 000000000..25b10f48b --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,136 @@ +# Quick Start Guide + +Get up and running with taskcat in just a few minutes! This guide will walk you through creating and running your first taskcat test. + +## Step 1: Create a Simple Template + +First, let's create a basic CloudFormation template to test: + +```yaml +# templates/simple-s3.yaml +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Simple S3 bucket for taskcat testing' + +Parameters: + BucketName: + Type: String + Description: Name for the S3 bucket + Default: my-test-bucket + +Resources: + TestBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub "${BucketName}-${AWS::Region}-${AWS::AccountId}" + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + +Outputs: + BucketName: + Description: Name of the created bucket + Value: !Ref TestBucket + Export: + Name: !Sub "${AWS::StackName}-BucketName" +``` + +## Step 2: Create taskcat Configuration + +Create a taskcat configuration file: + +```yaml +# .taskcat.yml +project: + name: my-first-taskcat-test + regions: + - us-east-1 + - us-west-2 + +tests: + simple-test: + template: templates/simple-s3.yaml + parameters: + BucketName: $[taskcat_random-string] +``` + +## Step 3: Run Your First Test + +Execute the test: + +```bash +taskcat test run +``` + +taskcat will: +1. 🚀 Deploy your template in specified regions +2. ✅ Validate the deployment +3. 📊 Generate a detailed report +4. 🧹 Clean up resources + +## Step 4: View Results + +Check the results in the `taskcat_outputs` directory: + +```bash +ls taskcat_outputs/ +# index.html - Main report +# logs/ - Detailed logs +# templates/ - Processed templates +``` + +Open `taskcat_outputs/index.html` in your browser to see the visual report. + +## What Just Happened? + +taskcat performed these actions: + +1. **Template Processing**: Replaced pseudo-parameters with actual values +2. **Multi-Region Deployment**: Created CloudFormation stacks in us-east-1 and us-west-2 +3. **Validation**: Verified successful deployment and resource creation +4. **Reporting**: Generated comprehensive HTML and JSON reports +5. **Cleanup**: Automatically deleted test resources + +## Next Steps + +Now that you've run your first test, explore: + +- [Configuration Guide](configuration.md) - Advanced configuration options +- [Dynamic Values](../usage/DYNAMIC_VALUES.md) - Runtime-evaluated parameters and AWS environment integration +- [Examples](../examples/) - Real-world usage scenarios + +## Common Next Actions + +### Test Multiple Templates +```yaml +tests: + test1: + template: templates/vpc.yaml + test2: + template: templates/ec2.yaml + parameters: + InstanceType: t3.micro +``` + +### Add Parameter Overrides +```yaml +tests: + production-test: + template: templates/app.yaml + parameters: + Environment: prod + InstanceType: m5.large +``` + +### Customize Regions +```yaml +project: + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 +``` + +Congratulations! You've successfully run your first taskcat test. 🎉 diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..2d2c443db --- /dev/null +++ b/docs/index.md @@ -0,0 +1,243 @@ +# taskcat Documentation + +
+ +

taskcat

+

Test your AWS CloudFormation templates across multiple regions with confidence. taskcat automates the deployment and validation of your infrastructure as code, ensuring your templates work reliably everywhere.

+
+ Get Started + Install Now +
+
+ +## What is taskcat? + +taskcat is a powerful testing framework for AWS CloudFormation templates that helps you validate your infrastructure as code across multiple AWS regions simultaneously. Built by AWS Solutions Architects, taskcat ensures your templates are robust, reliable, and ready for production deployment. + +
+
+

Multi-Region Testing

+

Deploy and test your CloudFormation templates across multiple AWS regions simultaneously to ensure global compatibility and resilience.

+
+ +
+

Automated Validation

+

Comprehensive automated testing with detailed pass/fail reporting, stack validation, and resource verification.

+
+ +
+

Dynamic Values

+

Runtime-evaluated parameters that pull values from your AWS environment, generate random data, and provide context-aware configurations.

+
+ +
+

Rich Reporting

+

Generate detailed HTML reports with deployment status, logs, and visual dashboards to track your testing results.

+
+ +
+

CI/CD Integration

+

Seamlessly integrate with your continuous integration pipelines using GitHub Actions, Jenkins, or AWS CodePipeline.

+
+
+ +## Key Features + +### 🚀 **Quick Setup** +Get started in minutes with simple configuration files and intuitive CLI commands. + +### 🌍 **Global Testing** +Test across all AWS regions or specify custom region sets for your deployment requirements. + +### ⚡ **Dynamic Values** +Runtime-evaluated parameters that can pull values from your AWS environment, generate random data, and provide context-aware configurations for flexible testing. + +### 📊 **Comprehensive Reports** +Generate detailed reports with stack outputs, resource details, and deployment timelines. + +### 🔒 **Security First** +Built-in security best practices with IAM role management and secure parameter handling. + +## Quick Start Example + +
+
Basic taskcat Configuration
+ +=== "taskcat.yml" + + ```yaml + project: + name: my-cloudformation-project + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + + tests: + default: + template: templates/main.yaml + parameters: + InstanceType: t3.micro + AvailabilityZones: $[taskcat_genaz_2] + DatabasePassword: $[taskcat_genpass_16S] + S3Bucket: $[taskcat_autobucket] + ``` + +=== "Advanced Configuration" + + ```yaml + project: + name: enterprise-app + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + + global: + parameters: + ProjectName: $[taskcat_project_name] + Environment: production + + tests: + vpc-infrastructure: + template: templates/vpc.yaml + parameters: + VpcName: $[taskcat_project_name]-vpc-$[taskcat_current_region] + AvailabilityZones: $[taskcat_genaz_3] + + database-tier: + template: templates/rds.yaml + parameters: + DBInstanceId: $[taskcat_project_name]-db-$[taskcat_genuuid] + MasterPassword: $[taskcat_secretsmanager_prod/db/password] + DBSubnetGroup: $[taskcat_getval_VpcName]-db-subnets + + application-tier: + template: templates/app.yaml + parameters: + AppName: $[taskcat_project_name]-app + S3Bucket: $[taskcat_autobucket] + ApiKey: $[taskcat_ssm_/app/api/key] + CurrentRegion: $[taskcat_current_region] + ``` + +=== "CLI Commands" + + ```bash + # Install taskcat + pip install taskcat + + # Initialize a new project + taskcat init + + # Test your templates + taskcat test run + + # Generate reports + taskcat test run --output-directory ./reports + ``` + +=== "GitHub Actions" + + ```yaml + name: taskcat Tests + on: [push, pull_request] + + jobs: + taskcat: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + - name: Install taskcat + run: pip install taskcat + - name: Run Tests + run: taskcat test run + ``` +
+ +## Why Choose taskcat? + +
+🏆 AWS Solutions Architecture Team Approved
+taskcat is developed and maintained by the AWS Solutions Architecture team and is used internally by AWS for testing CloudFormation templates in AWS Quick Starts and Solutions. +
+ +### **Proven at Scale** +- Used by AWS internally for testing hundreds of CloudFormation templates +- Powers the AWS Quick Start program with over 150+ tested solutions +- Trusted by enterprises worldwide for production deployments + +### **Developer Friendly** +- Simple YAML configuration +- Intuitive command-line interface +- Rich documentation and examples +- Active community support + +### **Enterprise Ready** +- Multi-account testing support +- Advanced parameter management +- Comprehensive logging and reporting +- Integration with AWS services + +## Getting Help + +
+
+
📚 Documentation
+

Comprehensive guides, tutorials, and API reference to help you get the most out of taskcat.

+ Browse Docs +
+ +
+
💬 Community
+

Join our community discussions, ask questions, and share your taskcat experiences.

+ Join Community +
+ +
+
🐛 Issues
+

Report bugs, request features, or contribute to the taskcat project on GitHub.

+ Report Issue +
+
+ +## What's New + +
+🎉 Latest Updates
+Check out the latest features including enhanced pseudo-parameters, improved AWS service integrations, and better CI/CD support. +
+ +### Recent Improvements +- **Enhanced Pseudo-Parameters**: New AWS service integrations and improved parameter handling +- **Better Error Reporting**: More detailed error messages and troubleshooting guidance +- **Performance Optimizations**: Faster template processing and parallel execution +- **Extended AWS Service Support**: Support for latest AWS services and regions + +## Next Steps + +Ready to start testing your CloudFormation templates? Here's your path forward: + +1. **[Install taskcat](getting-started/installation.md)** - Get taskcat up and running in minutes +2. **[Quick Start Guide](getting-started/quickstart.md)** - Run your first test +3. **[Configuration Guide](getting-started/configuration.md)** - Learn about advanced configuration options +4. **[Dynamic Values](usage/DYNAMIC_VALUES.md)** - Master runtime-evaluated parameters and AWS environment integration +5. **[Examples](examples/)** - Explore real-world usage scenarios + +
+💡 Pro Tip
+Start with the Quick Start guide to run your first test, then explore Dynamic Values to make your templates flexible with runtime-evaluated parameters that pull from your AWS environment. +
+ +--- + +
+

taskcat - Making CloudFormation testing simple, reliable, and scalable.

+

Built with ❤️ by the AWS Solutions Architecture Team

+
diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 000000000..da58c580a --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,146 @@ +# Installation + +Get taskcat installed and running on your system quickly and easily. + +## Prerequisites + +Before installing taskcat, ensure you have: + +- **Python 3.8+** - taskcat requires Python 3.8 or higher +- **AWS CLI** - Configured with appropriate credentials +- **Git** - For cloning repositories and version control +- **AWS Permissions** - See [Required Permissions](#required-permissions) + +## Installation Methods + +### PyPI (Recommended) + +Install taskcat from the Python Package Index: + +```bash +pip install taskcat +``` + +### Development Installation + +Install the latest development version: + +```bash +pip install git+https://github.com/aws-ia/taskcat.git +``` + +### Docker + +Use the official Docker image: + +```bash +# Pull the image +docker pull public.ecr.aws/aws-ia/taskcat:latest + +# Run taskcat in a container +docker run -it --rm \ + -v $(pwd):/workspace \ + -v ~/.aws:/root/.aws \ + public.ecr.aws/aws-ia/taskcat:latest --help +``` + +## Verification + +Verify your installation: + +```bash +# Check version +taskcat --version + +# Display help +taskcat --help + +# Test basic functionality +taskcat lint --help +``` + +## AWS Configuration + +### Configure Credentials + +taskcat uses AWS credentials from your environment. Choose one method: + +#### AWS CLI Configuration +```bash +aws configure +``` + +#### Environment Variables +```bash +export AWS_ACCESS_KEY_ID=your-access-key +export AWS_SECRET_ACCESS_KEY=your-secret-key +export AWS_DEFAULT_REGION=us-east-1 +``` + +#### IAM Roles (Recommended) +For EC2 instances or Lambda functions, taskcat automatically uses attached IAM roles. + +### Required Permissions + +taskcat requires these AWS permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "cloudformation:*", + "s3:*", + "iam:ListRoles", + "iam:PassRole", + "ec2:Describe*", + "logs:*" + ], + "Resource": "*" + } + ] +} +``` + +!!! warning "Security Note" + These are broad permissions for testing. In production, use more restrictive policies based on your specific templates. + +## Troubleshooting + +### Common Issues + +**Python version error:** +```bash +# Check Python version +python --version +# or +python3 --version +``` + +**Permission denied:** +```bash +# Install with user flag +pip install --user taskcat +``` + +**AWS credentials not found:** +```bash +# Verify AWS configuration +aws sts get-caller-identity +``` + +### Getting Help + +- Check the [Troubleshooting Guide](troubleshooting.md) +- Visit our [GitHub Issues](https://github.com/aws-ia/taskcat/issues) +- Join the community discussions + +## Next Steps + +Once installed, proceed to: + +1. [Quick Start](quickstart.md) - Run your first test +2. [Configuration](configuration.md) - Learn configuration options +3. [Dynamic Values](dynamic-values.md) - Master runtime parameters diff --git a/docs/parameter-overrides.md b/docs/parameter-overrides.md new file mode 100644 index 000000000..f14864707 --- /dev/null +++ b/docs/parameter-overrides.md @@ -0,0 +1,532 @@ +# Parameter Overrides + +Learn how to override CloudFormation parameters at different levels for flexible and reusable test configurations. + +## Overview + +Parameter overrides allow you to: + +- **Reuse templates** across different environments +- **Customize deployments** without modifying templates +- **Manage configurations** hierarchically +- **Test variations** of the same template + +## Override Hierarchy + +Parameters are resolved in this order (highest to lowest priority): + +1. **Test-level parameters** - Specific to individual tests +2. **Project-level parameters** - Applied to all project tests +3. **Global parameters** - Applied to all tests +4. **Template defaults** - Default values in CloudFormation template + +```yaml +general: + parameters: + Environment: test # Lowest priority + +project: + parameters: + Environment: staging # Overrides global + +tests: + production: + parameters: + Environment: prod # Highest priority +``` + +## Basic Parameter Overrides + +### Template with Parameters + +```yaml +# template.yaml +AWSTemplateFormatVersion: '2010-09-09' +Parameters: + InstanceType: + Type: String + Default: t3.micro + Environment: + Type: String + Default: dev + DatabasePassword: + Type: String + NoEcho: true +``` + +### Configuration with Overrides + +```yaml +# .taskcat.yml +project: + name: parameter-override-example + parameters: + Environment: staging # Override template default + +tests: + development: + template: template.yaml + parameters: + InstanceType: t3.micro + Environment: dev # Override project setting + + production: + template: template.yaml + parameters: + InstanceType: m5.large + Environment: prod # Override project setting + DatabasePassword: $[taskcat_genpass_20S] +``` + +## Global Parameter Overrides + +Set parameters that apply to all tests: + +```yaml +general: + parameters: + # Common parameters for all tests + ProjectName: my-application + Owner: platform-team + CostCenter: "1001" + + tags: + # Common tags for all stacks + Project: my-application + ManagedBy: taskcat +``` + +## Project Parameter Overrides + +Set parameters that apply to all tests in a project: + +```yaml +project: + name: web-application + parameters: + # Project-wide parameters + ApplicationName: web-app + Environment: staging + VpcCidr: 10.0.0.0/16 + +tests: + vpc-test: + template: templates/vpc.yaml + # Inherits all project parameters + + app-test: + template: templates/app.yaml + parameters: + Environment: production # Overrides project Environment +``` + +## Test-Specific Overrides + +Override parameters for individual tests: + +```yaml +tests: + small-deployment: + template: templates/app.yaml + parameters: + InstanceType: t3.micro + MinSize: 1 + MaxSize: 2 + + large-deployment: + template: templates/app.yaml + parameters: + InstanceType: m5.xlarge + MinSize: 3 + MaxSize: 10 + + secure-deployment: + template: templates/app.yaml + parameters: + InstanceType: m5.large + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + EnableEncryption: true +``` + +## Environment-Based Overrides + +Create environment-specific configurations: + +```yaml +project: + name: multi-environment-app + +tests: + development: + template: templates/app.yaml + parameters: + Environment: dev + InstanceType: t3.micro + DatabaseInstanceClass: db.t3.micro + BackupRetentionPeriod: 1 + MultiAZ: false + + staging: + template: templates/app.yaml + parameters: + Environment: staging + InstanceType: t3.medium + DatabaseInstanceClass: db.t3.small + BackupRetentionPeriod: 7 + MultiAZ: false + + production: + template: templates/app.yaml + parameters: + Environment: prod + InstanceType: m5.large + DatabaseInstanceClass: db.r5.large + BackupRetentionPeriod: 30 + MultiAZ: true + EnableEncryption: true + EnableMonitoring: true +``` + +## Region-Specific Overrides + +Different parameters for different regions: + +```yaml +tests: + multi-region-app: + template: templates/app.yaml + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + parameters: + # Base parameters for all regions + InstanceType: m5.large + Environment: prod + + us-east-1-specific: + template: templates/app.yaml + regions: + - us-east-1 + parameters: + # US East specific configuration + InstanceType: m5.xlarge + EnableCloudFront: true + PrimaryRegion: true + + europe-specific: + template: templates/app.yaml + regions: + - eu-west-1 + parameters: + # Europe specific configuration + InstanceType: m5.large + DataResidency: EU + ComplianceMode: GDPR +``` + +## Dynamic Parameter Overrides + +Combine static and dynamic parameters: + +```yaml +project: + parameters: + # Static project parameters + ProjectName: my-app + Owner: development-team + +tests: + dynamic-test: + template: templates/app.yaml + parameters: + # Static overrides + Environment: production + InstanceType: m5.large + + # Dynamic values + S3Bucket: $[taskcat_autobucket] + DatabasePassword: $[taskcat_genpass_20S] + UniqueId: $[taskcat_genuuid] + CurrentRegion: $[taskcat_current_region] + + # Context-aware values + StackName: $[taskcat_project_name]-$[taskcat_test_name] + LogGroup: /aws/lambda/$[taskcat_project_name] +``` + +## Complex Parameter Scenarios + +### Conditional Parameters + +```yaml +tests: + conditional-test: + template: templates/conditional.yaml + parameters: + CreateDatabase: true + DatabaseInstanceClass: db.t3.micro + DatabasePassword: $[taskcat_genpass_16S] + + no-database-test: + template: templates/conditional.yaml + parameters: + CreateDatabase: false + # Database parameters not needed +``` + +### Nested Stack Parameters + +```yaml +tests: + nested-stack-test: + template: templates/parent.yaml + parameters: + # Parent stack parameters + Environment: production + + # Child stack parameters (passed through) + VpcTemplateUrl: https://s3.amazonaws.com/templates/vpc.yaml + VpcInstanceType: t3.medium + + AppTemplateUrl: https://s3.amazonaws.com/templates/app.yaml + AppInstanceType: m5.large + AppDatabasePassword: $[taskcat_genpass_20S] +``` + +### Cross-Stack References + +```yaml +tests: + vpc-stack: + template: templates/vpc.yaml + parameters: + VpcCidr: 10.0.0.0/16 + Environment: production + + app-stack: + template: templates/app.yaml + parameters: + # Reference outputs from vpc-stack + VpcId: $[taskcat_getval_VpcId] + PrivateSubnets: $[taskcat_getval_PrivateSubnets] + Environment: production + InstanceType: m5.large +``` + +## Parameter File Integration + +Use external parameter files: + +### Parameter File + +```json +// parameters/production.json +[ + { + "ParameterKey": "InstanceType", + "ParameterValue": "m5.large" + }, + { + "ParameterKey": "Environment", + "ParameterValue": "production" + }, + { + "ParameterKey": "DatabaseInstanceClass", + "ParameterValue": "db.r5.large" + } +] +``` + +### Configuration Reference + +```yaml +tests: + production-from-file: + template: templates/app.yaml + parameter_input: parameters/production.json + parameters: + # Additional parameters or overrides + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] +``` + +## Validation and Testing + +### Parameter Validation + +```bash +# Validate parameters before testing +taskcat lint + +# Test with specific parameters +taskcat test run --parameters InstanceType=t3.micro,Environment=test +``` + +### Parameter Debugging + +```yaml +tests: + debug-parameters: + template: templates/debug.yaml + parameters: + # Use outputs to verify parameter values + DebugInstanceType: t3.micro + DebugEnvironment: $[taskcat_test_name] + DebugRegion: $[taskcat_current_region] + DebugProject: $[taskcat_project_name] +``` + +## Best Practices + +### 1. Use Hierarchical Configuration + +```yaml +# ✅ Good: Logical hierarchy +general: + parameters: + Owner: platform-team # Global default + +project: + parameters: + Environment: staging # Project default + +tests: + production: + parameters: + Environment: prod # Test-specific override +``` + +### 2. Group Related Parameters + +```yaml +# ✅ Good: Grouped by function +tests: + web-app: + parameters: + # Instance configuration + InstanceType: m5.large + MinSize: 2 + MaxSize: 10 + + # Database configuration + DatabaseInstanceClass: db.r5.large + DatabasePassword: $[taskcat_genpass_20S] + MultiAZ: true + + # Security configuration + EnableEncryption: true + SSLCertificateArn: $[taskcat_ssm_/ssl/cert/arn] +``` + +### 3. Use Meaningful Parameter Names + +```yaml +# ✅ Good: Descriptive names +parameters: + WebServerInstanceType: m5.large + DatabaseMasterPassword: $[taskcat_genpass_20S] + ApplicationLoadBalancerScheme: internet-facing + +# ❌ Avoid: Generic names +parameters: + Type1: m5.large + Password: $[taskcat_genpass_20S] + Scheme: internet-facing +``` + +### 4. Document Parameter Purpose + +```yaml +tests: + documented-test: + template: templates/app.yaml + parameters: + # Production-grade instance for high availability + InstanceType: m5.large + + # Secure password for RDS master user + DatabasePassword: $[taskcat_genpass_20S] + + # Enable encryption for compliance requirements + EnableEncryption: true +``` + +## Common Patterns + +### Multi-Tier Application + +```yaml +project: + name: three-tier-app + parameters: + ProjectName: three-tier-app + Environment: production + +tests: + web-tier: + template: templates/web-tier.yaml + parameters: + InstanceType: m5.large + MinSize: 2 + MaxSize: 10 + + app-tier: + template: templates/app-tier.yaml + parameters: + InstanceType: m5.xlarge + MinSize: 3 + MaxSize: 15 + + data-tier: + template: templates/data-tier.yaml + parameters: + DatabaseInstanceClass: db.r5.2xlarge + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + MultiAZ: true +``` + +### Blue-Green Deployment + +```yaml +tests: + blue-environment: + template: templates/app.yaml + parameters: + Environment: blue + InstanceType: m5.large + LoadBalancerWeight: 100 + + green-environment: + template: templates/app.yaml + parameters: + Environment: green + InstanceType: m5.large + LoadBalancerWeight: 0 +``` + +## Troubleshooting + +### Common Issues + +**Parameter not found:** +- Verify parameter exists in template +- Check parameter name spelling +- Ensure parameter is not marked as `NoEcho` + +**Type mismatch:** +- Verify parameter type in template +- Check value format (string, number, boolean) +- Validate array/list formatting + +**Override not working:** +- Check parameter hierarchy +- Verify configuration syntax +- Test with `taskcat lint` + +For more help, see the [Troubleshooting Guide](troubleshooting.md). + +## Next Steps + +- [Dynamic Values](dynamic-values.md) - Runtime-evaluated parameters +- [Configuration Guide](configuration.md) - Complete configuration options +- [Schema Reference](schema.md) - Full parameter reference diff --git a/docs/preso/_content.html b/docs/preso/_content.html deleted file mode 100644 index 24b3866cd..000000000 --- a/docs/preso/_content.html +++ /dev/null @@ -1,171 +0,0 @@ -## What is taskcat? -**taskcat** is a tool that tests AWS CloudFormation templates. It deploys your AWS -CloudFormation template in multiple AWS Regions and generates a report with a pass/fail -grade for each region. You can specify the regions and number of Availability Zones you -want to include in the test, and pass in parameter values from your AWS CloudFormation -template. taskcat is implemented as a Python class that you import, instantiate, and run. - -taskcat was developed by the aws-ia team to test AWS CloudFormation templates -that automatically deploy workloads on AWS. We’re pleased to make the tool available to -all developers who want to validate their custom AWS CloudFormation templates across -AWS Regions ---- -### CLI -The cli is self documenting by using `--help`. The most common use of taskcat is for -executing function tests of CloudFormation templates. The command for this is: - -```bash -taskcat test run -``` - -add `--help to see the supported flags and arguments` ---- - -```bash - -optional arguments: - -h, --help show this help message and exit - -v, --version show program's version number and exit - -q, --quiet reduce output to the minimum - -d, --debug adds debug output and tracebacks - --profile _PROFILE set the default profile used. - -commands: - delete - [ALPHA] Deletes an installed package in an AWS account/region - deploy - [ALPHA] installs a stack into an AWS account/region - lint - checks CloudFormation templates for issues using cfn-python-lint - list - [ALPHA] lists taskcat jobs with active stacks - package - packages lambda source files into zip files. If a dockerfile is present in asource folder, it will be run prior to zipping the contents - test - Performs functional tests on CloudFormation templates. - update-ami - Updates AMI IDs within CloudFormation templates - upload - Uploads project to S3. -``` - -### Python -Taskcat can be imported into Python and used in the testing framework of your choice. -``` python -from taskcat.testing import CFNTest -test = CFNTest.from_file(project_root='./template_dir') -with test as stacks: - # Calling 'with' or 'test.run()' will deploy the stacks. - for stack in stacks: - print(f"Testing {stack.name}") - bucket_name = "" - for output in stack.outputs: - if output.key == "LogsBucketName": - bucket_name = output.value - break - assert "logs" in bucket_name - assert stack.region.name in bucket_name - print(f"Created bucket: {bucket_name}") -``` ---- - -### Config files -taskcat has several configuration files which can be used to set behaviors in a flexible way. - -#### Global config -`~/.taskcat.yml` provides global settings that become defaults for all projects. Please see our [schema reference](docs/schema/taskcat_schema.html) for specific configuration options that are available. - -#### Project config -`/.taskcat.yml` provides project specific configuration. Please see our [schema reference](docs/schema/taskcat_schema.html) for specific configuration options that are available. ---- -#### Precedence - -`~/.taskcat.yml` -```yaml -general: - s3_bucket: my-globally-defined-bucket - parameters: - KeyPair: my-global-ec2-keypair -``` ---- - -Given a simple project config: - -```yaml -project: - name: my-project - regions: - - us-east-2 -tests: - default: - template: ./template.yaml -``` ---- - -The effective test configuration would become: - -```yaml -tests: - default: - template: ./template.yaml - s3_bucket: my-globally-defined-bucket - parameters: - KeyPair: my-global-ec2-keypair -``` - -If any item is re-defined in a project it takes precedence over the global value. -Anything defined in a test takes precedence over what is defined in the project or -global configuration. with the **exception** of the `parameters` section which works in -reverse. For example, using the same global config as above, given this project config: - --- - -### Parameter overrides - -Parameter Overrides were added to the taskcat to solve a couple of common problems. First, many -templates share common parameters that are unique to an AWS account, like a KeyPair name -or an S3 Bucket, overrides provided a way to store those centrally for all your projects. -Second, we didn't want to add sensitive data (usernames, passwords, tokens) to a git -repository. The idea was to store sensitive/unique data outside of a git repository, but still -execute a test using this data. To that end, *any parameter defined in the global config -will take precedence over the same parameter in a project-level config*. --- - - -| Psuedo-Parameter | Example Value passed to the CloudFormation stack | Details | -| ------------- | ------------- | ------------- | -| `$[taskcat_autobucket]` | taskcat-tag-sample-taskcat-project-5fba6597 | _Note: The S3 Bucket is created_ | -| `$[taskcat_genaz_1]` | "us-east-1a" | Fetches a single Availability Zone within the region being launched in | -| `$[taskcat_genaz_2]` | "us-east-1a,us-east-1b" | Fetches two AvailabilityZones within the region being launched in | -| `$[taskcat_genaz_3]` | "us-east-1a,us-east-1b,us-east-1c" | Fetches three AvailabilityZones within the region being launched in | -| `$[taskcat_genpass_8A]` | tI8zN3iX8 | An alphanumberic 8-charater random password. The length is customizable. | -| `$[taskcat_genpass_8S]` | mA5@cB5! | An alphanumberic 8-charater random password. The length is customizable. | -| `$[taskcat_random-string]` | yysuawpwubvotiqgwjcu | Generates a random string | -| `$[taskcat_random-numbers]` | 56188163597280820763 | Generates random numbers. | -| `$[taskcat_genuuid]` | 1c2e3483-2c99-45bb-801d-8af68a3b907b | Generates a UUID | -| `$[taskcat_getval_MyAppPassword]` | _Dynamically generated password for the MyAppPassword parameter_ | Retreives another parameter value.| -| $[taskcat_current_region] | "us-east-2" | Region the test is being prepared for | -| $[taskcat_project_name] | "my-example-project" | Name of the project being tested | -| $[taskcat_test_name] | "cluster-with-windows-ad" | Name of the test being tested | -| $[taskcat_ssm_/path/to/ssm/parameter] | _SSM Parameter Value_ | Retreives values from SSM | -| $[taskcat_secretsmanager_SecretNameOrARN] |_Value from SecretsManager_ | Retreives a secret value from SecretsManager given an name or ARN| ---- -#### From: (defined in taskcat.yaml') -``` - InstanceType: t2.small - AvailablityZones: $[taskcat_genaz_2] - RandomString: $[taskcat_random-string] - RandomNumbers: $[taskcat_random-numbers] - GenerateUUID: $[taskcat_genuuid] - Password: $[taskcat_genpass_8A] - PasswordConfirm: $[taskcat_getval_Password] -``` ---- -#### To: (At runtime passed to cloudformation API) -``` - InstanceType: t2.small - AvailablityZones: us-east-1a: us-east1b - RandomString: yysuawpwubvotiqgwjcu - RandomNumbers: 56188163597280820763 - GenerateUUID: 1c2e3483-2c99-45bb-801d-8af68a3b907b - Password: tI8zN3iX8 - PasswordConfirm: tI8zN3iX8 -``` ---- - -Questions? ---- -Thanks! ---- diff --git a/docs/preso/_footer.html b/docs/preso/_footer.html deleted file mode 100644 index ac2afa39f..000000000 --- a/docs/preso/_footer.html +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - diff --git a/docs/preso/_header.html b/docs/preso/_header.html deleted file mode 100644 index 7630f5fba..000000000 --- a/docs/preso/_header.html +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - - Remark - - - - - - - - - - diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 000000000..440190e43 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,190 @@ +# Quick Start + +Get up and running with taskcat in under 5 minutes. This guide walks you through creating and running your first test. + +## Step 1: Create a CloudFormation Template + +Create a simple template to test: + +```yaml +# template.yaml +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Simple S3 bucket for taskcat testing' + +Parameters: + BucketName: + Type: String + Description: Name for the S3 bucket + Default: my-test-bucket + +Resources: + TestBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub "${BucketName}-${AWS::AccountId}-${AWS::Region}" + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + +Outputs: + BucketName: + Description: Name of the created S3 bucket + Value: !Ref TestBucket + BucketArn: + Description: ARN of the created S3 bucket + Value: !GetAtt TestBucket.Arn +``` + +## Step 2: Create taskcat Configuration + +Create a `.taskcat.yml` configuration file: + +```yaml +# .taskcat.yml +project: + name: my-first-test + regions: + - us-east-1 + - us-west-2 + +tests: + basic-test: + template: template.yaml + parameters: + BucketName: $[taskcat_random-string] +``` + +## Step 3: Run Your First Test + +Execute the test: + +```bash +taskcat test run +``` + +taskcat will: + +1. 🚀 **Deploy** your template in specified regions +2. ✅ **Validate** the deployment succeeds +3. 📊 **Generate** a detailed report +4. 🧹 **Clean up** all test resources + +## Step 4: View Results + +Check the results: + +```bash +# List output files +ls taskcat_outputs/ + +# Open the HTML report +open taskcat_outputs/index.html +``` + +The report includes: +- Deployment status for each region +- CloudFormation events and logs +- Resource details and outputs +- Performance metrics + +## What Just Happened? + +taskcat performed these actions: + +1. **Template Processing** - Replaced `$[taskcat_random-string]` with a unique value +2. **Multi-Region Deployment** - Created CloudFormation stacks in us-east-1 and us-west-2 +3. **Validation** - Verified successful deployment and resource creation +4. **Reporting** - Generated comprehensive HTML and JSON reports +5. **Cleanup** - Automatically deleted all test resources + +## Next Steps + +Now that you've run your first test, explore: + +### Advanced Configuration +```yaml +tests: + production-test: + template: template.yaml + parameters: + BucketName: $[taskcat_project_name]-prod + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + + development-test: + template: template.yaml + parameters: + BucketName: $[taskcat_project_name]-dev + regions: + - us-east-1 +``` + +### Multiple Templates +```yaml +tests: + vpc-test: + template: templates/vpc.yaml + + app-test: + template: templates/application.yaml + parameters: + InstanceType: t3.micro +``` + +### Dynamic Parameters +```yaml +tests: + secure-test: + template: template.yaml + parameters: + BucketName: $[taskcat_autobucket] + DatabasePassword: $[taskcat_genpass_16S] + AvailabilityZones: $[taskcat_genaz_2] + CurrentRegion: $[taskcat_current_region] +``` + +## Common Commands + +```bash +# Test specific configuration +taskcat test run --config-file custom.yml + +# Test specific regions +taskcat test run --regions us-east-1,us-west-2 + +# Keep resources after testing (for debugging) +taskcat test run --no-delete + +# Lint configuration before testing +taskcat lint + +# List available tests +taskcat test list +``` + +## Troubleshooting + +**Test fails with permission errors:** +- Verify AWS credentials: `aws sts get-caller-identity` +- Check IAM permissions for CloudFormation and S3 + +**Template validation errors:** +- Run `taskcat lint` to check configuration +- Validate CloudFormation template syntax + +**Resources not cleaned up:** +- Check CloudFormation console for failed deletions +- Manually delete stuck stacks if needed + +## Learn More + +- [Configuration Guide](configuration.md) - Advanced configuration options +- [Dynamic Values](dynamic-values.md) - Runtime-evaluated parameters +- [Examples](examples.md) - Real-world usage scenarios +- [Schema Reference](schema.md) - Complete configuration reference + +Congratulations! You've successfully run your first taskcat test. 🎉 diff --git a/docs/readme.amiupdater.md b/docs/readme.amiupdater.md deleted file mode 100644 index 7c355cbc9..000000000 --- a/docs/readme.amiupdater.md +++ /dev/null @@ -1,127 +0,0 @@ - - -# *AMIUpdater README* - - - -## General Usage. - -`amiupdater ` - -For a current list of options, see.. - -`amiupdater -h` - -## Leveraging the Upstream Config File - -### Upstream Mappings - -By default, AMIUpdater uses a config file bundled with `taskcat`. This config file is populated with common AMI Mappings, such as *Amazon Linux AMI* and *Ubuntu Server 18.04*. To see all of the mappings available, [check out the config file](https://github.com/aws-ia/taskcat/blob/master/taskcat/cfg/amiupdater.cfg.yml) - -To utilize these upstream mappings, simply leverage them in your templates. - -_Note: The AMI IDs are here for example purposes. When first configuring the Mapping, you can filll them with arbitrary data._ - -- JSON - -```json -{ -(...) - "Mappings": { - "AWSAMIRegionMap": { - "ap-northeast-1": { - "AMZNLINUXHVM": "ami-00a5245b4816c38e6", - "CENTOS7HVM": "ami-8e8847f1", - "US1404HVM": "ami-0be9269b44d4b26c1", - "US1604HVM": "ami-0d5e82481c5fd4ad5", - "SLES15HVM": "ami-09161bc9964f46a98" - }, - "ap-northeast-2": { - "AMZNLINUXHVM": "ami-00dc207f8ba6dc919", - "CENTOS7HVM": "ami-bf9c36d1", - "US1404HVM": "ami-017332df4b882edd2", - "US1604HVM": "ami-0507b772e2c9b8c15", - "SLES15HVM": "ami-04ecb44b7d8e8d354" - }, - "ap-south-1": { - "AMZNLINUXHVM": "ami-0ad42f4f66f6c1cc9", - "CENTOS7HVM": "ami-1780a878", - "US1404HVM": "ami-09dcf5653a185f5df", - "US1604HVM": "ami-0c8810f694cbe10ba", - "SLES15HVM": "ami-025d8258d76079367" - } - (...) - } - } - } -} -``` - -- YAML -```yaml -Mappings: - AWSAMIRegionMap: - ap-northeast-1: - AMZNLINUXHVM: ami-00a5245b4816c38e6, - CENTOS7HVM: ami-8e8847f1, - US1404HVM: ami-0be9269b44d4b26c1, - US1604HVM: ami-0d5e82481c5fd4ad5, - SLES15HVM: ami-09161bc9964f46a98 - ap-northeast-2: - AMZNLINUXHVM: ami-00dc207f8ba6dc919, - CENTOS7HVM: ami-bf9c36d1, - US1404HVM: ami-017332df4b882edd2, - US1604HVM: ami-0507b772e2c9b8c15, - SLES15HVM: ami-04ecb44b7d8e8d354 - ap-south-1: - AMZNLINUXHVM: ami-0ad42f4f66f6c1cc9, - CENTOS7HVM: ami-1780a878, - US1404HVM: ami-09dcf5653a185f5df, - US1604HVM: ami-0c8810f694cbe10ba, - SLES15HVM: ami-025d8258d76079367 -``` -## Defining your own AMI Mappings - -### Custom Config File -Functionally the same as the upstream config file, a local config file can be created and used in deployment pipelines. - -For a full list of filters, available, [please see the AWS EC2 API Documentation](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html). - -```yaml -# Owner-id must be in quotes -# Whereas, all other filters do not need quotes, -# because they are not in a number format - -global: - AMIs: - CUSTOM_MAPPING_1: - name: my_super_awesome_name-* - owner-id: 1234567890 - CUSTOM_MAPPING_2: - name: my_super_other_awesome_name ???? * - owner-id: 1234567890 - architecture: arm64 -``` - -### Template Inline Config -- JSON -```json - "Metadata": { - "AWSAMIRegionMap":{ - "Filters":{ - "":{ - "name":"my awesome AMI NAME", - "owner-id":"01234567890" - } - } - } -``` -- YAML -```yaml -Metadata: - AWSAMIRegionMap: - Filters: - : - name: my awesome AMI NAME - owner-id: 01234567890 -``` diff --git a/docs/schema.md b/docs/schema.md new file mode 100644 index 000000000..0094b7e2e --- /dev/null +++ b/docs/schema.md @@ -0,0 +1,308 @@ +# Schema Reference + +The taskcat configuration schema defines the structure and validation rules for `.taskcat.yml` files. This reference provides a comprehensive overview of all available configuration options. + +## Overview + +The taskcat configuration file uses YAML format and supports three main sections: + +- **`general`** - Global configuration settings +- **`project`** - Project-specific configuration +- **`tests`** - Individual test definitions + +## Configuration Structure + +```yaml +# Basic structure +general: + # Global settings applied to all tests + +project: + # Project-specific settings + +tests: + test-name: + # Individual test configuration +``` + +## General Section + +Global configuration settings that apply to all tests unless overridden. + +### Properties + +| Property | Type | Description | Example | +|----------|------|-------------|---------| +| `artifact_regions` | `array[string]` | AWS regions where artifacts are copied | `["us-east-1", "us-west-2"]` | +| `auth` | `object` | AWS authentication profiles by region | `{"default": "my-profile"}` | +| `parameters` | `object` | Global CloudFormation parameters | `{"InstanceType": "t3.micro"}` | +| `regions` | `array[string]` | Default AWS regions for testing | `["us-east-1", "us-west-2"]` | +| `s3_bucket` | `string` | S3 bucket for artifacts (auto-generated if omitted) | `"my-taskcat-bucket"` | +| `s3_regional_buckets` | `boolean` | Enable regional auto-buckets | `true` | +| `tags` | `object` | CloudFormation stack tags | `{"Environment": "test"}` | +| `prehooks` | `array[object]` | Hooks executed before tests | See [Hooks](#hooks) | +| `posthooks` | `array[object]` | Hooks executed after tests | See [Hooks](#hooks) | + +### Example + +```yaml +general: + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + parameters: + Environment: test + Owner: taskcat-team + tags: + Project: taskcat + CostCenter: "1001" + s3_regional_buckets: true +``` + +## Project Section + +Project-specific configuration that applies to all tests within the project. + +### Properties + +| Property | Type | Description | Default | +|----------|------|-------------|---------| +| `name` | `string` | Project name (lowercase, hyphens only) | - | +| `owner` | `string` | Project owner email | - | +| `regions` | `array[string]` | Project default regions | - | +| `s3_bucket` | `string` | Project S3 bucket name | Auto-generated | +| `s3_object_acl` | `string` | S3 object ACL | `"private"` | +| `template` | `string` | Default template path | - | +| `parameters` | `object` | Project default parameters | `{}` | +| `auth` | `object` | Project authentication settings | - | +| `lambda_source_path` | `string` | Lambda source directory | `"lambda_functions/source"` | +| `lambda_zip_path` | `string` | Lambda zip output directory | `"lambda_functions/packages"` | +| `package_lambda` | `boolean` | Enable Lambda packaging | `true` | +| `build_submodules` | `boolean` | Build Lambda zips for submodules | `true` | +| `shorten_stack_name` | `boolean` | Use shortened stack names | `false` | +| `role_name` | `string` | IAM role for CloudFormation | - | +| `org_id` | `string` | AWS Organization ID | - | +| `az_blacklist` | `array[string]` | Excluded Availability Zone IDs | `[]` | + +### Example + +```yaml +project: + name: my-cloudformation-project + owner: developer@example.com + regions: + - us-east-1 + - us-west-2 + parameters: + ProjectName: my-project + Environment: production + lambda_source_path: functions/source + lambda_zip_path: functions/packages + package_lambda: true + shorten_stack_name: true +``` + +## Tests Section + +Individual test configurations. Each test can override project and global settings. + +### Properties + +| Property | Type | Description | Required | +|----------|------|-------------|----------| +| `template` | `string` | CloudFormation template path | ✅ | +| `parameters` | `object` | Test-specific parameters | - | +| `regions` | `array[string]` | Test-specific regions | - | +| `auth` | `object` | Test-specific authentication | - | +| `artifact_regions` | `array[string]` | Test-specific artifact regions | - | +| `az_blacklist` | `array[string]` | Test-specific AZ exclusions | - | + +### Example + +```yaml +tests: + basic-test: + template: templates/basic.yaml + parameters: + InstanceType: t3.micro + KeyName: my-key-pair + regions: + - us-east-1 + - us-west-2 + + advanced-test: + template: templates/advanced.yaml + parameters: + InstanceType: m5.large + DatabasePassword: $[taskcat_genpass_16S] + S3Bucket: $[taskcat_autobucket] + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + auth: + us-east-1: production-profile + eu-west-1: europe-profile +``` + +## Parameter Types + +CloudFormation parameters support multiple data types: + +### String Parameters +```yaml +parameters: + InstanceType: t3.micro + KeyName: my-key-pair +``` + +### Numeric Parameters +```yaml +parameters: + Port: 8080 + MaxSize: 10 +``` + +### Boolean Parameters +```yaml +parameters: + EnableLogging: true + CreateDatabase: false +``` + +### Array Parameters +```yaml +parameters: + SecurityGroups: + - sg-12345678 + - sg-87654321 + AvailabilityZones: + - us-east-1a + - us-east-1b +``` + +## Hooks + +Hooks allow you to execute custom scripts before (`prehooks`) or after (`posthooks`) test execution. + +### Hook Structure + +```yaml +prehooks: + - type: script + config: + command: ./scripts/setup.sh + +posthooks: + - type: script + config: + command: ./scripts/cleanup.sh +``` + +### Hook Types + +| Type | Description | Configuration | +|------|-------------|---------------| +| `script` | Execute shell script | `command`: Script path or command | + +## AWS Regions + +Valid AWS region formats follow the pattern: `^(ap|eu|us|sa|ca|cn|af|me|us-gov)-(central|south|north|east|west|southeast|southwest|northeast|northwest)-[0-9]$` + +### Examples +- `us-east-1` - US East (N. Virginia) +- `us-west-2` - US West (Oregon) +- `eu-west-1` - Europe (Ireland) +- `ap-southeast-1` - Asia Pacific (Singapore) + +## Availability Zone IDs + +When using `az_blacklist`, specify Availability Zone IDs (not names): + +### Examples +- `use1-az1` - US East 1 AZ 1 +- `usw2-az2` - US West 2 AZ 2 +- `euw1-az3` - EU West 1 AZ 3 + +## S3 Object ACLs + +Valid S3 object ACL values: + +- `private` (default) +- `public-read` +- `public-read-write` +- `authenticated-read` +- `aws-exec-read` +- `bucket-owner-read` +- `bucket-owner-full-control` + +## Complete Example + +```yaml +# Complete taskcat configuration example +general: + regions: + - us-east-1 + - us-west-2 + parameters: + Environment: test + tags: + Project: taskcat-example + Owner: development-team + +project: + name: example-project + owner: developer@example.com + s3_regional_buckets: true + package_lambda: true + lambda_source_path: functions/source + lambda_zip_path: functions/packages + +tests: + vpc-test: + template: templates/vpc.yaml + parameters: + VpcCidr: 10.0.0.0/16 + AvailabilityZones: $[taskcat_genaz_2] + regions: + - us-east-1 + - us-west-2 + + application-test: + template: templates/application.yaml + parameters: + InstanceType: t3.medium + DatabasePassword: $[taskcat_genpass_20S] + S3Bucket: $[taskcat_autobucket] + KeyName: $[taskcat_getkeypair] + regions: + - us-east-1 + auth: + us-east-1: production-profile +``` + +## Validation + +The schema includes built-in validation for: + +- **Required fields** - Ensures essential properties are present +- **Data types** - Validates correct data types for each property +- **Format validation** - Checks AWS region formats, naming patterns +- **Enum validation** - Validates against allowed values (e.g., S3 ACLs) +- **Pattern matching** - Ensures strings match required patterns + +## Best Practices + +1. **Use descriptive test names** - Make test purposes clear +2. **Leverage inheritance** - Define common settings in `general` or `project` +3. **Use Dynamic Values** - Employ `$[taskcat_*]` functions for flexibility +4. **Regional considerations** - Test in multiple regions for global deployments +5. **Parameter validation** - Ensure all required parameters are provided +6. **Resource cleanup** - Use appropriate hooks for setup and teardown + +## See Also + +- [Dynamic Values Reference](dynamic-values.md) - Complete guide to `$[taskcat_*]` functions +- [Configuration Guide](configuration.md) - Detailed configuration examples +- [Parameter Overrides](parameter-overrides.md) - Advanced parameter techniques diff --git a/docs/schema/schema_doc.css b/docs/schema/schema_doc.css deleted file mode 100644 index cd1fcd358..000000000 --- a/docs/schema/schema_doc.css +++ /dev/null @@ -1,180 +0,0 @@ -body { - font: 16px/1.5em "Overpass", "Open Sans", Helvetica, sans-serif; - color: #333; - font-weight: 300; - padding: 40px; -} - -.btn.btn-link { - font-size: 18px; -} - -.jsfh-animated-property { - animation: eclair; - animation-iteration-count: 1; - animation-fill-mode: forwards; - animation-duration: .75s; - -} - -@keyframes eclair { - 0%,100% { - transform: scale(1); - } - 50% { - transform: scale(1.03); - } -} - -.btn.btn-primary { - margin: 10px; -} - -.btn.example-show.collapsed:before { - content: "show" -} - -.btn.example-show:before { - content: "hide" -} - -.description.collapse:not(.show) { - max-height: 100px !important; - overflow: hidden; - - display: -webkit-box; - -webkit-line-clamp: 2; - -webkit-box-orient: vertical; -} - -.description.collapsing { - min-height: 100px !important; -} - -.collapse-description-link.collapsed:after { - content: '+ Read More'; -} - -.collapse-description-link:not(.collapsed):after { - content: '- Read Less'; -} - -.badge { - font-size: 100%; - margin-bottom: 0.5rem; - margin-top: 0.5rem; -} - -.badge.value-type { - font-size: 120%; - margin-right: 5px; - margin-bottom: 10px; -} - - -.badge.default-value { - font-size: 120%; - margin-left: 5px; - margin-bottom: 10px; -} - -.badge.restriction { - display: inline-block; -} - -.badge.required-property,.badge.deprecated-property,.badge.pattern-property,.badge.no-additional { - font-size: 100%; - margin-left: 10px; -} - -.accordion div.card:only-child { - border-bottom: 1px solid rgba(0, 0, 0, 0.125); -} - -.examples { - padding: 1rem !important; -} - -.examples pre { - margin-bottom: 0; -} - -.highlight.jumbotron { - padding: 1rem !important; -} - -.generated-by-footer { - margin-top: 1em; - text-align: right; -} - -/* From https://github.com/richleland/pygments-css/blob/master/friendly.css, see https://github.com/trentm/python-markdown2/wiki/fenced-code-blocks */ -.highlight { background: #e9ecef; } /* Changed from #f0f0f0 in the original style to be the same as bootstrap's jumbotron */ -.highlight .hll { background-color: #ffffcc } -.highlight .c { color: #60a0b0; font-style: italic } /* Comment */ -.highlight .err { border: 1px solid #FF0000 } /* Error */ -.highlight .k { color: #007020; font-weight: bold } /* Keyword */ -.highlight .o { color: #666666 } /* Operator */ -.highlight .ch { color: #60a0b0; font-style: italic } /* Comment.Hashbang */ -.highlight .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */ -.highlight .cp { color: #007020 } /* Comment.Preproc */ -.highlight .cpf { color: #60a0b0; font-style: italic } /* Comment.PreprocFile */ -.highlight .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */ -.highlight .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */ -.highlight .gd { color: #A00000 } /* Generic.Deleted */ -.highlight .ge { font-style: italic } /* Generic.Emph */ -.highlight .gr { color: #FF0000 } /* Generic.Error */ -.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ -.highlight .gi { color: #00A000 } /* Generic.Inserted */ -.highlight .go { color: #888888 } /* Generic.Output */ -.highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ -.highlight .gs { font-weight: bold } /* Generic.Strong */ -.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ -.highlight .gt { color: #0044DD } /* Generic.Traceback */ -.highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ -.highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ -.highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ -.highlight .kp { color: #007020 } /* Keyword.Pseudo */ -.highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ -.highlight .kt { color: #902000 } /* Keyword.Type */ -.highlight .m { color: #40a070 } /* Literal.Number */ -.highlight .s { color: #4070a0 } /* Literal.String */ -.highlight .na { color: #4070a0 } /* Name.Attribute */ -.highlight .nb { color: #007020 } /* Name.Builtin */ -.highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ -.highlight .no { color: #60add5 } /* Name.Constant */ -.highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ -.highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ -.highlight .ne { color: #007020 } /* Name.Exception */ -.highlight .nf { color: #06287e } /* Name.Function */ -.highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ -.highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ -.highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ -.highlight .nv { color: #bb60d5 } /* Name.Variable */ -.highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ -.highlight .w { color: #bbbbbb } /* Text.Whitespace */ -.highlight .mb { color: #40a070 } /* Literal.Number.Bin */ -.highlight .mf { color: #40a070 } /* Literal.Number.Float */ -.highlight .mh { color: #40a070 } /* Literal.Number.Hex */ -.highlight .mi { color: #40a070 } /* Literal.Number.Integer */ -.highlight .mo { color: #40a070 } /* Literal.Number.Oct */ -.highlight .sa { color: #4070a0 } /* Literal.String.Affix */ -.highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ -.highlight .sc { color: #4070a0 } /* Literal.String.Char */ -.highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */ -.highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ -.highlight .s2 { color: #4070a0 } /* Literal.String.Double */ -.highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ -.highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ -.highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ -.highlight .sx { color: #c65d09 } /* Literal.String.Other */ -.highlight .sr { color: #235388 } /* Literal.String.Regex */ -.highlight .s1 { color: #4070a0 } /* Literal.String.Single */ -.highlight .ss { color: #517918 } /* Literal.String.Symbol */ -.highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ -.highlight .fm { color: #06287e } /* Name.Function.Magic */ -.highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ -.highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ -.highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ -.highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */ -.highlight .il { color: #40a070 } /* Literal.Number.Integer.Long */ diff --git a/docs/schema/schema_doc.min.js b/docs/schema/schema_doc.min.js deleted file mode 100644 index 58eb60040..000000000 --- a/docs/schema/schema_doc.min.js +++ /dev/null @@ -1 +0,0 @@ -function flashElement(t){myElement=document.getElementById(t),myElement.classList.add("jsfh-animated-property"),setTimeout(function(){myElement.classList.remove("jsfh-animated-property")},1e3)}function setAnchor(t){history.pushState({},"",t)}function anchorOnLoad(){let t=window.location.hash.split("?")[0].split("&")[0];"#"===t[0]&&(t=t.substr(1)),t.length>0&&anchorLink(t)}function anchorLink(t){$("#"+t).parents().addBack().filter(".collapse:not(.show), .tab-pane, [role='tab']").each(function(t){if($(this).hasClass("collapse"))$(this).collapse("show");else if($(this).hasClass("tab-pane")){const t=$("a[href='#"+$(this).attr("id")+"']");t&&t.tab("show")}else"tab"===$(this).attr("role")&&$(this).tab("show")}),setTimeout(function(){let e=document.getElementById(t);e&&(e.scrollIntoView({block:"center",behavior:"smooth"}),setTimeout(function(){flashElement(t)},500))},1e3)}$(document).on("click",'a[href^="#"]',function(t){t.preventDefault(),history.pushState({},"",this.href)}); diff --git a/docs/support/troubleshooting.md b/docs/support/troubleshooting.md new file mode 100644 index 000000000..df1890bb9 --- /dev/null +++ b/docs/support/troubleshooting.md @@ -0,0 +1,408 @@ +# Troubleshooting Guide + +Having issues with taskcat? This comprehensive troubleshooting guide will help you diagnose and resolve common problems. + +## 🔍 Quick Diagnostics + +Before diving into specific issues, run these diagnostic commands: + +```bash +# Check taskcat version and installation +taskcat --version + +# Validate your configuration +taskcat test run --dry-run + +# Check AWS credentials +aws sts get-caller-identity + +# Verify CloudFormation permissions +aws cloudformation describe-stacks --region us-east-1 +``` + +## 🚨 Common Issues + +### Installation Problems + +#### Python Version Issues +**Problem**: `taskcat requires Python 3.8 or higher` + +**Solution**: +```bash +# Check Python version +python3 --version + +# Install Python 3.8+ if needed +# macOS: brew install python@3.9 +# Ubuntu: sudo apt-get install python3.9 +# Windows: Download from python.org +``` + +#### Permission Denied During Installation +**Problem**: `Permission denied` when running `pip install taskcat` + +**Solution**: +```bash +# Install for current user only +pip install --user taskcat + +# Or use virtual environment (recommended) +python3 -m venv taskcat-env +source taskcat-env/bin/activate # Linux/macOS +# taskcat-env\Scripts\activate # Windows +pip install taskcat +``` + +### AWS Credentials Issues + +#### No Credentials Found +**Problem**: `Unable to locate credentials` + +**Solutions**: +```bash +# Option 1: Configure AWS CLI +aws configure + +# Option 2: Set environment variables +export AWS_ACCESS_KEY_ID=your-access-key +export AWS_SECRET_ACCESS_KEY=your-secret-key +export AWS_DEFAULT_REGION=us-east-1 + +# Option 3: Use IAM roles (for EC2/Lambda) +# Attach appropriate IAM role to your instance +``` + +#### Insufficient Permissions +**Problem**: `Access Denied` errors during testing + +**Required Permissions**: +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "cloudformation:*", + "s3:*", + "iam:PassRole", + "iam:CreateRole", + "iam:DeleteRole", + "iam:GetRole", + "iam:AttachRolePolicy", + "iam:DetachRolePolicy", + "ec2:Describe*", + "ssm:GetParameter*", + "secretsmanager:GetSecretValue" + ], + "Resource": "*" + } + ] +} +``` + +### Configuration Issues + +#### Invalid YAML Syntax +**Problem**: `YAML syntax error in .taskcat.yml` + +**Solution**: +```bash +# Validate YAML syntax +python3 -c "import yaml; yaml.safe_load(open('.taskcat.yml'))" + +# Common issues: +# - Incorrect indentation (use spaces, not tabs) +# - Missing colons after keys +# - Unquoted special characters +``` + +**Example of correct syntax**: +```yaml +project: + name: my-project # ✅ Correct + regions: + - us-east-1 # ✅ Correct indentation + - us-west-2 + +tests: + test1: + template: templates/main.yaml # ✅ Quoted path with spaces + parameters: + Key: "Value with spaces" # ✅ Quoted value +``` + +#### Template Not Found +**Problem**: `Template file not found` + +**Solutions**: +```bash +# Check file path +ls -la templates/ + +# Use relative paths from project root +template: templates/main.yaml # ✅ Correct +template: ./templates/main.yaml # ✅ Also correct +template: /full/path/main.yaml # ❌ Avoid absolute paths +``` + +### Template Issues + +#### CloudFormation Validation Errors +**Problem**: Template fails CloudFormation validation + +**Debugging Steps**: +```bash +# Validate template syntax +aws cloudformation validate-template \ + --template-body file://templates/main.yaml + +# Check for common issues: +# - Invalid resource types +# - Missing required properties +# - Circular dependencies +# - Invalid parameter constraints +``` + +#### Parameter Issues +**Problem**: `Parameter validation failed` + +**Common Causes**: +- Missing required parameters +- Invalid parameter values +- Type mismatches +- Constraint violations + +**Solution**: +```yaml +# Ensure all required parameters are provided +tests: + test1: + template: templates/main.yaml + parameters: + RequiredParam1: value1 + RequiredParam2: value2 +``` + +### Pseudo-Parameter Issues + +#### Git Branch Parameter Fails +**Problem**: `Project root is not a git repository` + +**Solution**: +```bash +# Initialize git repository +git init +git add . +git commit -m "Initial commit" + +# Or avoid using $[taskcat_git_branch] +``` + +#### Availability Zone Issues +**Problem**: `Not enough availability zones in region` + +**Solutions**: +```yaml +# Reduce number of AZs requested +parameters: + AvailabilityZones: $[taskcat_genaz_2] # Instead of _3 or higher + +# Or test in regions with more AZs +project: + regions: + - us-east-1 # Has 6 AZs + - us-west-2 # Has 4 AZs +``` + +#### SSM Parameter Not Found +**Problem**: `Parameter /path/to/param not found` + +**Solutions**: +```bash +# Verify parameter exists +aws ssm get-parameter --name "/path/to/param" --region us-east-1 + +# Check parameter path spelling +parameters: + AMI: $[taskcat_ssm_/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2] +``` + +### Deployment Issues + +#### Stack Creation Fails +**Problem**: CloudFormation stack creation fails + +**Debugging Steps**: +1. Check CloudFormation console for detailed error messages +2. Review stack events for failure points +3. Validate resource limits and quotas +4. Check for naming conflicts + +```bash +# View stack events +aws cloudformation describe-stack-events \ + --stack-name your-stack-name \ + --region us-east-1 +``` + +#### Resource Limits Exceeded +**Problem**: `LimitExceeded` errors + +**Solutions**: +- Check AWS service limits +- Use different instance types +- Test in fewer regions simultaneously +- Clean up unused resources + +#### Timeout Issues +**Problem**: Stack creation times out + +**Solutions**: +```yaml +# Increase timeout in template +Resources: + MyResource: + Type: AWS::EC2::Instance + CreationPolicy: + ResourceSignal: + Timeout: PT15M # 15 minutes +``` + +### Performance Issues + +#### Slow Test Execution +**Problem**: Tests take too long to complete + +**Optimization Strategies**: +```yaml +# Test in fewer regions initially +project: + regions: + - us-east-1 # Start with one region + +# Use smaller instance types +parameters: + InstanceType: t3.nano # Faster launch times + +# Parallel execution (default) +# taskcat runs tests in parallel automatically +``` + +#### Memory Issues +**Problem**: taskcat runs out of memory + +**Solutions**: +```bash +# Increase available memory +# For Docker: docker run -m 4g taskcat + +# Reduce concurrent tests +# Split large test suites into smaller batches +``` + +## 🔧 Advanced Debugging + +### Enable Debug Logging +```bash +# Enable verbose logging +taskcat test run --debug + +# Save logs to file +taskcat test run --debug > taskcat-debug.log 2>&1 +``` + +### Template Preprocessing Debug +```bash +# See processed templates +taskcat test run --keep-failed + +# Check generated parameters +ls taskcat_outputs/ +cat taskcat_outputs/*/parameters.json +``` + +### AWS CloudTrail Integration +```bash +# Monitor AWS API calls +aws logs filter-log-events \ + --log-group-name CloudTrail/taskcatTesting \ + --start-time $(date -d '1 hour ago' +%s)000 +``` + +## 🆘 Getting Help + +### Before Asking for Help + +1. **Check this troubleshooting guide** +2. **Search existing GitHub issues** +3. **Enable debug logging** +4. **Gather system information**: + ```bash + taskcat --version + python3 --version + aws --version + uname -a # Linux/macOS + ``` + +### Where to Get Help + +#### GitHub Issues +- **Bug Reports**: [Create an issue](https://github.com/aws-ia/taskcat/issues/new?template=bug_report.md) +- **Feature Requests**: [Request a feature](https://github.com/aws-ia/taskcat/issues/new?template=feature_request.md) +- **Questions**: [Ask a question](https://github.com/aws-ia/taskcat/discussions) + +#### Community Support +- **AWS re:Post**: Tag questions with `taskcat` +- **Stack Overflow**: Use the `taskcat` tag +- **AWS Forums**: CloudFormation section + +### Creating Effective Bug Reports + +Include this information: + +```markdown +## Environment +- taskcat version: X.X.X +- Python version: X.X.X +- Operating System: OS name and version +- AWS CLI version: X.X.X + +## Configuration +```yaml +# Your .taskcat.yml (remove sensitive data) +``` + +## Template +```yaml +# Minimal template that reproduces the issue +``` + +## Error Output +``` +# Full error message and stack trace +``` + +## Steps to Reproduce +1. Step one +2. Step two +3. Step three +``` + +## 📚 Additional Resources + +- [AWS CloudFormation User Guide](https://docs.aws.amazon.com/cloudformation/) +- [AWS CLI Configuration](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) +- [taskcat GitHub Repository](https://github.com/aws-ia/taskcat) +- [AWS Quick Start Program](https://aws.amazon.com/quickstart/) + +## 🔄 Still Having Issues? + +If this guide doesn't resolve your issue: + +1. **Search GitHub Issues**: Someone might have encountered the same problem +2. **Check AWS Service Health**: Verify AWS services are operational +3. **Try a Minimal Example**: Isolate the problem with a simple test case +4. **Contact Support**: Create a detailed GitHub issue with all relevant information + +Remember: The more information you provide, the faster we can help resolve your issue! 🚀 diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 000000000..bb8132681 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,513 @@ +# Troubleshooting + +Common issues and solutions for taskcat configuration and execution problems. + +## Installation Issues + +### Python Version Compatibility + +**Problem:** `taskcat requires Python 3.8 or higher` + +**Solution:** +```bash +# Check Python version +python --version +python3 --version + +# Install Python 3.8+ if needed +# macOS with Homebrew +brew install python@3.9 + +# Ubuntu/Debian +sudo apt update +sudo apt install python3.9 + +# Update pip and reinstall +pip3 install --upgrade pip +pip3 install taskcat +``` + +### Permission Denied Errors + +**Problem:** `Permission denied` during installation + +**Solution:** +```bash +# Install with user flag +pip install --user taskcat + +# Or use virtual environment +python -m venv taskcat-env +source taskcat-env/bin/activate # Linux/macOS +# taskcat-env\Scripts\activate # Windows +pip install taskcat +``` + +### Package Not Found + +**Problem:** `No module named 'taskcat'` + +**Solution:** +```bash +# Verify installation +pip list | grep taskcat + +# Reinstall if missing +pip uninstall taskcat +pip install taskcat + +# Check PATH +echo $PATH +which taskcat +``` + +## AWS Configuration Issues + +### Credentials Not Found + +**Problem:** `Unable to locate credentials` + +**Solution:** +```bash +# Check AWS configuration +aws configure list +aws sts get-caller-identity + +# Configure credentials +aws configure + +# Or set environment variables +export AWS_ACCESS_KEY_ID=your-access-key +export AWS_SECRET_ACCESS_KEY=your-secret-key +export AWS_DEFAULT_REGION=us-east-1 +``` + +### Insufficient Permissions + +**Problem:** `Access Denied` or `User is not authorized` + +**Solution:** +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "cloudformation:*", + "s3:*", + "iam:ListRoles", + "iam:PassRole", + "ec2:Describe*", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "*" + } + ] +} +``` + +### Region Not Available + +**Problem:** `Region not supported` or `Service not available` + +**Solution:** +```yaml +# Check available regions +tests: + region-test: + template: template.yaml + regions: + - us-east-1 # Always available + - us-west-2 # Good alternative + # Remove unsupported regions +``` + +## Configuration Issues + +### YAML Syntax Errors + +**Problem:** `YAML parsing error` or `Invalid configuration` + +**Solution:** +```bash +# Validate YAML syntax +python -c "import yaml; yaml.safe_load(open('.taskcat.yml'))" + +# Use proper indentation (spaces, not tabs) +# Check for special characters +# Validate with taskcat lint +taskcat lint +``` + +### Template Not Found + +**Problem:** `Template file not found` + +**Solution:** +```yaml +# Use correct relative paths +tests: + my-test: + template: templates/app.yaml # Relative to .taskcat.yml + # Not: /full/path/to/template.yaml +``` + +```bash +# Verify file exists +ls -la templates/ +ls -la templates/app.yaml +``` + +### Parameter Validation Errors + +**Problem:** `Parameter validation failed` + +**Solution:** +```yaml +# Check parameter types and constraints +parameters: + InstanceType: t3.micro # String + Port: 8080 # Number + EnableLogging: true # Boolean + SecurityGroups: # Array + - sg-12345678 + - sg-87654321 +``` + +## Dynamic Values Issues + +### Dynamic Value Not Replaced + +**Problem:** `$[taskcat_genpass_16S]` appears in CloudFormation + +**Solution:** +```yaml +# Check syntax - must be exact +parameters: + Password: $[taskcat_genpass_16S] # ✅ Correct + # Password: $[taskcat_genpass_16s] # ❌ Wrong case + # Password: ${taskcat_genpass_16S} # ❌ Wrong brackets +``` + +### AWS Service Integration Fails + +**Problem:** `$[taskcat_ssm_/path]` returns error + +**Solution:** +```bash +# Verify parameter exists +aws ssm get-parameter --name /path/to/parameter + +# Check permissions +aws iam get-user +aws iam list-attached-user-policies --user-name your-username + +# Verify region +aws ssm get-parameter --name /path/to/parameter --region us-east-1 +``` + +### Availability Zone Issues + +**Problem:** `Not enough availability zones` + +**Solution:** +```yaml +# Use fewer AZs or check region support +parameters: + AvailabilityZones: $[taskcat_genaz_2] # Instead of 3+ + +# Or exclude problematic AZs +project: + az_blacklist: + - use1-az3 # Exclude specific AZ +``` + +## CloudFormation Issues + +### Stack Creation Fails + +**Problem:** Stack fails to create resources + +**Solution:** +```bash +# Check CloudFormation events +aws cloudformation describe-stack-events --stack-name stack-name + +# Validate template +aws cloudformation validate-template --template-body file://template.yaml + +# Test with minimal parameters +taskcat test run --no-delete # Keep resources for debugging +``` + +### Resource Limits Exceeded + +**Problem:** `LimitExceeded` errors + +**Solution:** +```bash +# Check service limits +aws service-quotas list-service-quotas --service-code ec2 + +# Use different regions +# Request limit increases +# Use smaller instance types for testing +``` + +### Dependency Issues + +**Problem:** Resources created in wrong order + +**Solution:** +```yaml +# Add explicit dependencies in template +Resources: + MyInstance: + Type: AWS::EC2::Instance + DependsOn: MySecurityGroup + Properties: + SecurityGroupIds: + - !Ref MySecurityGroup +``` + +## Test Execution Issues + +### Tests Hang or Timeout + +**Problem:** Tests never complete + +**Solution:** +```bash +# Check CloudFormation console for stuck stacks +# Increase timeout (if available) +# Cancel and retry +taskcat test run --no-delete # Debug mode + +# Clean up manually if needed +aws cloudformation delete-stack --stack-name stuck-stack +``` + +### Cleanup Failures + +**Problem:** Resources not deleted after test + +**Solution:** +```bash +# Check for deletion protection +aws cloudformation describe-stacks --stack-name stack-name + +# Manual cleanup +aws cloudformation delete-stack --stack-name stack-name + +# Force delete if needed (be careful!) +aws s3 rm s3://bucket-name --recursive +aws s3 rb s3://bucket-name +``` + +### Multiple Region Failures + +**Problem:** Some regions fail, others succeed + +**Solution:** +```yaml +# Test regions individually +tests: + us-east-1-test: + template: template.yaml + regions: + - us-east-1 + + us-west-2-test: + template: template.yaml + regions: + - us-west-2 + +# Check region-specific issues +# Verify service availability +# Check quotas per region +``` + +## Performance Issues + +### Slow Test Execution + +**Problem:** Tests take too long + +**Solution:** +```yaml +# Reduce regions for testing +regions: + - us-east-1 # Single region for development + +# Use smaller resources +parameters: + InstanceType: t3.nano # Smallest for testing + +# Parallel execution (if supported) +# Use simpler templates for initial testing +``` + +### S3 Upload Issues + +**Problem:** Template upload fails + +**Solution:** +```bash +# Check S3 permissions +aws s3 ls s3://your-bucket/ + +# Verify bucket exists and is accessible +aws s3 mb s3://your-taskcat-bucket + +# Check file sizes (CloudFormation limits) +ls -lh templates/ +``` + +## Debugging Techniques + +### Enable Verbose Logging + +```bash +# Run with debug output +taskcat test run --debug + +# Check log files +ls -la taskcat_outputs/ +cat taskcat_outputs/taskcat.log +``` + +### Validate Before Testing + +```bash +# Lint configuration +taskcat lint + +# Validate templates +aws cloudformation validate-template --template-body file://template.yaml + +# Test parameters +taskcat test run --no-delete +``` + +### Incremental Testing + +```yaml +# Start simple +tests: + minimal: + template: minimal-template.yaml + regions: + - us-east-1 + parameters: + InstanceType: t3.nano + +# Add complexity gradually +tests: + basic: + template: basic-template.yaml + # Add more parameters + + advanced: + template: full-template.yaml + # Full configuration +``` + +## Common Error Messages + +### `Template format error` + +**Cause:** Invalid CloudFormation template syntax + +**Solution:** +- Validate YAML/JSON syntax +- Check CloudFormation template structure +- Verify resource types and properties + +### `Parameter validation failed` + +**Cause:** Parameter doesn't match template constraints + +**Solution:** +- Check parameter types in template +- Verify allowed values +- Ensure required parameters are provided + +### `Stack already exists` + +**Cause:** Previous test didn't clean up + +**Solution:** +```bash +# Delete existing stack +aws cloudformation delete-stack --stack-name existing-stack + +# Wait for deletion +aws cloudformation wait stack-delete-complete --stack-name existing-stack +``` + +### `Bucket already exists` + +**Cause:** S3 bucket name conflict + +**Solution:** +```yaml +# Use dynamic bucket names +parameters: + BucketName: $[taskcat_autobucket] # Always unique + # Not: BucketName: my-static-bucket-name +``` + +## Getting Help + +### Check Documentation + +- [Configuration Guide](configuration.md) +- [Dynamic Values](dynamic-values.md) +- [Schema Reference](schema.md) + +### Community Resources + +- [GitHub Issues](https://github.com/aws-ia/taskcat/issues) +- [AWS re:Post](https://repost.aws/) +- [Stack Overflow](https://stackoverflow.com/questions/tagged/taskcat) + +### Debug Information to Collect + +When reporting issues, include: + +```bash +# System information +taskcat --version +python --version +aws --version + +# Configuration +cat .taskcat.yml + +# Error logs +cat taskcat_outputs/taskcat.log + +# CloudFormation events +aws cloudformation describe-stack-events --stack-name failing-stack +``` + +### Create Minimal Reproduction + +```yaml +# Minimal .taskcat.yml that reproduces the issue +project: + name: debug-issue + regions: + - us-east-1 + +tests: + reproduce-issue: + template: minimal-template.yaml + parameters: + TestParameter: test-value +``` + +This troubleshooting guide covers the most common issues encountered when using taskcat. For additional help, consult the community resources or create a detailed issue report with the debugging information outlined above. diff --git a/docs/usage/DYNAMIC_VALUES.md b/docs/usage/DYNAMIC_VALUES.md new file mode 100644 index 000000000..1434bbf88 --- /dev/null +++ b/docs/usage/DYNAMIC_VALUES.md @@ -0,0 +1,334 @@ +# Dynamic Values + +Dynamic Values are runtime-evaluated parameters that provide flexible, context-aware configurations for your CloudFormation templates. These values are evaluated during taskcat execution and can pull data from your AWS environment, generate random values, or provide contextual information about your test run. + +## Overview + +Dynamic Values solve common testing challenges: + +- **Environment-specific values**: Pull actual values from your AWS environment +- **Unique resource names**: Generate random strings to avoid naming conflicts +- **Context awareness**: Access current region, project name, and test information +- **Security**: Generate secure passwords and retrieve secrets safely +- **Flexibility**: Reference other parameters and create complex configurations + +## Syntax + +Dynamic Values use the syntax: `$[taskcat_function_name]` or `$[taskcat_function_name_parameter]` + +```yaml +parameters: + DatabasePassword: $[taskcat_genpass_16S] + S3BucketName: $[taskcat_autobucket] + CurrentRegion: $[taskcat_current_region] + AvailabilityZones: $[taskcat_genaz_2] +``` + +## Complete Dynamic Values Reference + +### Random Value Generation + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_random-string]` | Generate 20-character random string | `kj8s9dkf7h3m2n4p5q6r` | Unique resource identifiers | +| `$[taskcat_random-numbers]` | Generate 20-digit random number | `12345678901234567890` | Unique numeric identifiers | +| `$[taskcat_genuuid]` | Generate UUID v1 | `550e8400-e29b-41d4-a716-446655440000` | Globally unique identifiers | + +### Password Generation + +| Dynamic Value | Description | Example | Use Case | +|---------------|-------------|---------|----------| +| `$[taskcat_genpass_8]` | 8-character alphanumeric password | `aB3dE7gH` | Simple passwords | +| `$[taskcat_genpass_16S]` | 16-character password with special chars | `aB3!dE7@gH9#kL2$` | Secure passwords | +| `$[taskcat_genpass_32A]` | 32-character alphanumeric password | `aB3dE7gH9kL2mN4pQ6rS8tU0vW2xY4zA` | Long secure passwords | + +**Password Types:** +- No suffix: Alphanumeric only +- `S`: Includes special characters (!@#$%^&*) +- `A`: Alphanumeric only (explicit) + +### AWS Environment Values + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_current_region]` | Current AWS region | `us-east-1` | Region-specific configurations | +| `$[taskcat_genaz_2]` | 2 availability zones | `us-east-1a,us-east-1b` | Multi-AZ deployments | +| `$[taskcat_genaz_3]` | 3 availability zones | `us-east-1a,us-east-1b,us-east-1c` | High availability setups | +| `$[taskcat_gensingleaz_1]` | Single AZ (1st available) | `us-east-1a` | Single AZ deployments | +| `$[taskcat_gensingleaz_2]` | Single AZ (2nd available) | `us-east-1b` | Specific AZ selection | + +### S3 and Storage + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_autobucket]` | Auto-generated S3 bucket name | `tcat-myproject-us-east-1-123456789` | Template artifacts | +| `$[taskcat_autobucket_prefix]` | S3 bucket prefix | `myproject-us-east-1-123456789` | Custom bucket naming | + +### Context Information + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_project_name]` | Current project name | `my-cloudformation-project` | Tagging and naming | +| `$[taskcat_test_name]` | Current test name | `production-test` | Test identification | +| `$[taskcat_git_branch]` | Current Git branch | `feature/new-feature` | Branch-specific configs | + +### Parameter References + +| Dynamic Value | Description | Example | Use Case | +|---------------|-------------|---------|----------| +| `$[taskcat_getval_ParameterName]` | Reference another parameter | `$[taskcat_getval_DatabasePassword]` | Parameter dependencies | + +### AWS Services Integration + +| Dynamic Value | Description | Example | Use Case | +|---------------|-------------|---------|----------| +| `$[taskcat_ssm_/path/to/parameter]` | Retrieve SSM Parameter | `$[taskcat_ssm_/app/database/host]` | Configuration management | +| `$[taskcat_secretsmanager_secret-name]` | Retrieve Secrets Manager value | `$[taskcat_secretsmanager_prod/db/password]` | Secure credential retrieval | + +### Legacy/Specialized Values + +| Dynamic Value | Description | Example Output | Use Case | +|---------------|-------------|----------------|----------| +| `$[taskcat_getkeypair]` | Default key pair name | `cikey` | EC2 key pair reference | +| `$[taskcat_getlicensebucket]` | License bucket placeholder | `override_this` | License content storage | +| `$[taskcat_getmediabucket]` | Media bucket placeholder | `override_this` | Media content storage | + +## Advanced Examples + +### Multi-Tier Application + +```yaml +project: + name: multi-tier-app + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + +global: + parameters: + ProjectName: $[taskcat_project_name] + Environment: production + +tests: + vpc-infrastructure: + template: templates/vpc.yaml + parameters: + VpcName: $[taskcat_project_name]-vpc-$[taskcat_current_region] + AvailabilityZones: $[taskcat_genaz_3] + + database-tier: + template: templates/rds.yaml + parameters: + DBInstanceIdentifier: $[taskcat_project_name]-db-$[taskcat_random-string] + MasterUsername: admin + MasterUserPassword: $[taskcat_genpass_32S] + DBSubnetGroupName: $[taskcat_getval_VpcName]-db-subnets + + application-tier: + template: templates/app.yaml + parameters: + ApplicationName: $[taskcat_project_name]-app + InstanceType: m5.large + KeyName: $[taskcat_getkeypair] + S3Bucket: $[taskcat_autobucket] + DatabaseEndpoint: $[taskcat_getval_DBInstanceIdentifier] + + monitoring: + template: templates/monitoring.yaml + parameters: + DashboardName: $[taskcat_project_name]-$[taskcat_test_name]-dashboard + LogGroupName: /aws/lambda/$[taskcat_project_name] + AlertEmail: $[taskcat_ssm_/notifications/email] +``` + +### Environment-Specific Configuration + +```yaml +project: + name: environment-configs + regions: + - us-east-1 + +tests: + development: + template: templates/app.yaml + parameters: + Environment: dev + InstanceType: t3.micro + DatabasePassword: $[taskcat_genpass_16] + S3Bucket: $[taskcat_project_name]-dev-$[taskcat_current_region] + + staging: + template: templates/app.yaml + parameters: + Environment: staging + InstanceType: t3.small + DatabasePassword: $[taskcat_secretsmanager_staging/db/password] + S3Bucket: $[taskcat_project_name]-staging-$[taskcat_current_region] + + production: + template: templates/app.yaml + parameters: + Environment: prod + InstanceType: m5.large + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + S3Bucket: $[taskcat_project_name]-prod-$[taskcat_current_region] + BackupRetention: 30 + MonitoringEnabled: true +``` + +### Security-Focused Configuration + +```yaml +project: + name: secure-app + regions: + - us-east-1 + - us-west-2 + +tests: + secure-deployment: + template: templates/secure-app.yaml + parameters: + # Generate unique, secure passwords + DatabaseMasterPassword: $[taskcat_genpass_32S] + ApplicationSecret: $[taskcat_genpass_24S] + + # Use AWS Secrets Manager for production secrets + ApiKey: $[taskcat_secretsmanager_prod/api/key] + CertificateArn: $[taskcat_ssm_/ssl/certificate/arn] + + # Generate unique resource names + KMSKeyAlias: $[taskcat_project_name]-key-$[taskcat_genuuid] + S3BucketName: $[taskcat_autobucket] + + # Context-aware naming + LogGroupName: /aws/lambda/$[taskcat_project_name]-$[taskcat_current_region] + + # Reference other parameters + DatabasePasswordConfirm: $[taskcat_getval_DatabaseMasterPassword] +``` + +### Multi-Region Deployment + +```yaml +project: + name: global-app + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 + +tests: + global-infrastructure: + template: templates/global-app.yaml + parameters: + # Region-specific configurations + PrimaryRegion: us-east-1 + CurrentRegion: $[taskcat_current_region] + + # Generate region-specific AZs + AvailabilityZones: $[taskcat_genaz_2] + + # Unique naming per region + S3BucketName: $[taskcat_project_name]-$[taskcat_current_region]-$[taskcat_random-numbers] + + # Global unique identifiers + DeploymentId: $[taskcat_genuuid] + + # Branch-specific configurations + GitBranch: $[taskcat_git_branch] + + # Environment from SSM + Environment: $[taskcat_ssm_/global/environment] +``` + +## Best Practices + +### 1. Use Appropriate Value Types + +```yaml +# ✅ Good: Use specific types for specific purposes +parameters: + DatabasePassword: $[taskcat_genpass_16S] # Secure password + ResourceId: $[taskcat_genuuid] # Globally unique + BucketName: $[taskcat_autobucket] # S3-compliant naming + +# ❌ Avoid: Using generic values for specific purposes +parameters: + DatabasePassword: $[taskcat_random-string] # Not secure enough + ResourceId: $[taskcat_random-numbers] # May not be unique +``` + +### 2. Leverage Parameter References + +```yaml +# ✅ Good: Reference parameters to maintain consistency +parameters: + MasterPassword: $[taskcat_genpass_20S] + PasswordConfirm: $[taskcat_getval_MasterPassword] + +# ❌ Avoid: Generating separate values for related parameters +parameters: + MasterPassword: $[taskcat_genpass_20S] + PasswordConfirm: $[taskcat_genpass_20S] # Different values! +``` + +### 3. Use Context-Aware Naming + +```yaml +# ✅ Good: Include context in resource names +parameters: + LogGroup: /aws/lambda/$[taskcat_project_name]-$[taskcat_current_region] + S3Bucket: $[taskcat_project_name]-logs-$[taskcat_current_region] + +# ❌ Avoid: Generic naming that may conflict +parameters: + LogGroup: /aws/lambda/myapp + S3Bucket: myapp-logs +``` + +### 4. Secure Credential Management + +```yaml +# ✅ Good: Use AWS services for production secrets +parameters: + DatabasePassword: $[taskcat_secretsmanager_prod/db/password] + ApiKey: $[taskcat_ssm_/app/api/key] + +# ✅ Good: Generate secure passwords for testing +parameters: + TestPassword: $[taskcat_genpass_16S] + +# ❌ Avoid: Hardcoded secrets +parameters: + DatabasePassword: "hardcoded-password" +``` + +## Troubleshooting + +### Common Issues + +**Dynamic Value not replaced:** +- Check syntax: `$[taskcat_function_name]` +- Verify function name spelling +- Ensure proper parameter placement + +**AWS service integration fails:** +- Verify IAM permissions for SSM/Secrets Manager +- Check parameter/secret exists in target region +- Validate parameter path format + +**AZ generation fails:** +- Check if region has enough AZs +- Verify region is enabled in your account +- Consider AZ exclusions in configuration + +**Parameter reference fails:** +- Ensure referenced parameter exists +- Check parameter name spelling +- Verify parameter is defined before reference + +For more troubleshooting help, see the [Troubleshooting Guide](../support/troubleshooting.md). diff --git a/docs/usage/PSUEDO_PARAMETERS.md b/docs/usage/PSUEDO_PARAMETERS.md index 712f59df5..d36f113db 100644 --- a/docs/usage/PSUEDO_PARAMETERS.md +++ b/docs/usage/PSUEDO_PARAMETERS.md @@ -1,43 +1,180 @@ -To increase the flexibility of taskcat, we've built-in support for _psuedo-parameters_ that are transposed at runtime for actual values. - -Following table describes the supported **psuedo-parameters**. - -| Psuedo-Parameter | Example Value passed to the CloudFormation stack | Details | -| ------------- | ------------- | ------------- | -| `$[taskcat_autobucket]` | taskcat-tag-sample-taskcat-project-5fba6597 | _Note: The S3 Bucket is created_ | -| `$[taskcat_genaz_1]` | "us-east-1a" | Fetches a single Availability Zone within the region being launched in | -| `$[taskcat_genaz_2]` | "us-east-1a,us-east-1b" | Fetches two AvailabilityZones within the region being launched in | -| `$[taskcat_genaz_3]` | "us-east-1a,us-east-1b,us-east-1c" | Fetches three AvailabilityZones within the region being launched in | -| `$[taskcat_genpass_8A]` | tI8zN3iX8 | An alphanumeric 8-charater random password. The length is customizable. | -| `$[taskcat_genpass_8S]` | mA5@cB5! | An alphanumeric 8-charater random password. The length is customizable. | -| `$[taskcat_random-string]` | yysuawpwubvotiqgwjcu | Generates a random string | -| `$[taskcat_random-numbers]` | 56188163597280820763 | Generates random numbers. | -| `$[taskcat_genuuid]` | 1c2e3483-2c99-45bb-801d-8af68a3b907b | Generates a UUID | -| `$[taskcat_getval_MyAppPassword]` | _Dynamically generated password for the MyAppPassword parameter_ | Retreives another parameter value.| -| `$[taskcat_current_region]` | "us-east-2" | Region the test is being prepared for | -| `$[taskcat_project_name]` | "my-example-project" | Name of the project being tested | -| `$[taskcat_test_name]` | "cluster-with-windows-ad" | Name of the test being tested | -| `$[taskcat_ssm_/path/to/ssm/parameter]` | _SSM Parameter Value_ | Retreives values from SSM | -| `$[taskcat_secretsmanager_SecretNameOrARN]` |_Value from SecretsManager_ | Retreives a secret value from SecretsManager given an name or ARN| - -#### From: (defined in taskcat.yaml') +# Pseudo-Parameters + +To increase the flexibility of taskcat, we've built-in support for **pseudo-parameters** that are dynamically replaced at runtime with actual values. These parameters allow you to create more flexible and reusable CloudFormation templates without hardcoding environment-specific values. + +## How Pseudo-Parameters Work + +Pseudo-parameters use the syntax `$[parameter_name]` and are processed by taskcat before the CloudFormation template is deployed. They are replaced with actual values based on the current test context, AWS environment, or generated data. + +## Available Pseudo-Parameters + +### 🪣 S3 Bucket Parameters + +| Pseudo-Parameter | Example Value | Description | +|------------------|---------------|-------------| +| `$[taskcat_autobucket]` | `taskcat-tag-sample-taskcat-project-5fba6597` | Creates a unique S3 bucket for the test. The bucket is automatically created and managed by taskcat. | +| `$[taskcat_autobucket_prefix]` | `taskcat-tag-sample-taskcat-project` | Returns the prefix portion of the auto-generated bucket name (without the random suffix). | + +### 🌍 Availability Zone Parameters + +| Pseudo-Parameter | Example Value | Description | +|------------------|---------------|-------------| +| `$[taskcat_genaz_1]` | `"us-east-1a"` | Returns a single available Availability Zone in the current region. | +| `$[taskcat_genaz_2]` | `"us-east-1a,us-east-1b"` | Returns two available Availability Zones in the current region, comma-separated. | +| `$[taskcat_genaz_3]` | `"us-east-1a,us-east-1b,us-east-1c"` | Returns three available Availability Zones in the current region, comma-separated. | +| `$[taskcat_genaz_N]` | `"us-east-1a,us-east-1b,..."` | Returns N available Availability Zones (replace N with desired number). | +| `$[taskcat_gensingleaz_1]` | `"us-east-1a"` | Returns a single AZ by index (1-based). Useful for consistent AZ selection. | +| `$[taskcat_gensingleaz_2]` | `"us-east-1b"` | Returns the second AZ in the region. | + +### 🔐 Password Generation Parameters + +| Pseudo-Parameter | Example Value | Description | +|------------------|---------------|-------------| +| `$[taskcat_genpass_8A]` | `tI8zN3iX` | Generates an 8-character alphanumeric password (letters + numbers). | +| `$[taskcat_genpass_12A]` | `vGceIP8EHCmn` | Generates a 12-character alphanumeric password. | +| `$[taskcat_genpass_8S]` | `mA5@cB5!` | Generates an 8-character password with special characters. | +| `$[taskcat_genpass_16S]` | `kL9#nM2$pQ4&rS6*` | Generates a 16-character password with special characters. | + +**Password Types:** +- **A** = Alphanumeric (letters + numbers) +- **S** = Special characters (letters + numbers + symbols: `!#$&{*:[=,]-_%@+`) +- **Length** = Any number from 1-99 + +### 🎲 Random Data Generation + +| Pseudo-Parameter | Example Value | Description | +|------------------|---------------|-------------| +| `$[taskcat_random-string]` | `yysuawpwubvotiqgwjcu` | Generates a 20-character random lowercase string. | +| `$[taskcat_random-numbers]` | `56188163597280820763` | Generates a 20-digit random number string. | +| `$[taskcat_genuuid]` | `1c2e3483-2c99-45bb-801d-8af68a3b907b` | Generates a UUID (Universally Unique Identifier). | + +### 📍 Context Information Parameters + +| Pseudo-Parameter | Example Value | Description | +|------------------|---------------|-------------| +| `$[taskcat_current_region]` | `"us-east-2"` | Returns the AWS region where the current test is being executed. | +| `$[taskcat_project_name]` | `"my-example-project"` | Returns the name of the taskcat project being tested. | +| `$[taskcat_test_name]` | `"cluster-with-windows-ad"` | Returns the name of the specific test being executed. | +| `$[taskcat_git_branch]` | `"main"` | Returns the current Git branch name (requires project to be a Git repository). | + +### 🔗 Parameter Reference + +| Pseudo-Parameter | Example Value | Description | +|------------------|---------------|-------------| +| `$[taskcat_getval_ParameterName]` | `tI8zN3iX` | Retrieves the value of another parameter. Useful for password confirmation fields. | + +### ☁️ AWS Service Integration + +| Pseudo-Parameter | Example Value | Description | +|------------------|---------------|-------------| +| `$[taskcat_ssm_/path/to/parameter]` | `ami-12345678` | Retrieves a value from AWS Systems Manager Parameter Store. | +| `$[taskcat_secretsmanager_SecretName]` | `{"username":"admin","password":"secret"}` | Retrieves a secret value from AWS Secrets Manager using secret name or ARN. | + +### 🔧 Legacy/Deprecated Parameters + +| Pseudo-Parameter | Example Value | Description | +|------------------|---------------|-------------| +| `$[taskcat_getkeypair]` | `cikey` | Returns a default keypair name. **Deprecated** - use parameter overrides instead. | +| `$[taskcat_getlicensebucket]` | `override_this` | Placeholder for license bucket. **Deprecated** - use parameter overrides instead. | +| `$[taskcat_getmediabucket]` | `override_this` | Placeholder for media bucket. **Deprecated** - use parameter overrides instead. | + +## Usage Examples + +### Basic Configuration (taskcat.yml) +```yaml +project: + name: my-cloudformation-project + regions: + - us-east-1 + - us-west-2 + +tests: + default: + template: templates/main.yaml + parameters: + InstanceType: t3.micro + AvailabilityZones: $[taskcat_genaz_2] + DatabasePassword: $[taskcat_genpass_16S] + ConfirmPassword: $[taskcat_getval_DatabasePassword] + S3Bucket: $[taskcat_autobucket] + CurrentRegion: $[taskcat_current_region] + ProjectName: $[taskcat_project_name] + RandomIdentifier: $[taskcat_genuuid] +``` + +### Runtime Transformation +**Before (in taskcat.yml):** +```yaml +parameters: + InstanceType: t3.micro + AvailabilityZones: $[taskcat_genaz_2] + DatabasePassword: $[taskcat_genpass_16S] + ConfirmPassword: $[taskcat_getval_DatabasePassword] + S3Bucket: $[taskcat_autobucket] + AMIId: $[taskcat_ssm_/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2] +``` + +**After (passed to CloudFormation):** +```yaml +parameters: + InstanceType: t3.micro + AvailabilityZones: "us-east-1a,us-east-1b" + DatabasePassword: "kL9#nM2$pQ4&rS6*" + ConfirmPassword: "kL9#nM2$pQ4&rS6*" + S3Bucket: "taskcat-my-project-a1b2c3d4" + AMIId: "ami-0abcdef1234567890" +``` + +## Advanced Usage + +### SSM Parameter Store Integration +```yaml +parameters: + LatestAMI: $[taskcat_ssm_/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2] + DatabaseEndpoint: $[taskcat_ssm_/myapp/prod/database/endpoint] + APIKey: $[taskcat_ssm_/myapp/prod/api/key] +``` + +### Secrets Manager Integration ```yaml - InstanceType: t2.small - AvailablityZones: $[taskcat_genaz_2] - RandomString: $[taskcat_random-string] - RandomNumbers: $[taskcat_random-numbers] - GenerateUUID: $[taskcat_genuuid] - Password: $[taskcat_genpass_8A] - PasswordConfirm: $[taskcat_getval_Password] +parameters: + DatabaseCredentials: $[taskcat_secretsmanager_prod/database/credentials] + APISecret: $[taskcat_secretsmanager_arn:aws:secretsmanager:us-east-1:123456789012:secret:MySecret-AbCdEf] ``` -#### To: (At runtime passed to cloudformation API) +### Multi-AZ Deployment ```yaml - InstanceType: t2.small - AvailablityZones: us-east-1a: us-east1b - RandomString: yysuawpwubvotiqgwjcu - RandomNumbers: 56188163597280820763 - GenerateUUID: 1c2e3483-2c99-45bb-801d-8af68a3b907b - Password: tI8zN3iX8 - PasswordConfirm: tI8zN3iX8 +parameters: + PublicSubnetAZs: $[taskcat_genaz_2] # For public subnets + PrivateSubnetAZs: $[taskcat_genaz_2] # Same AZs for private subnets + DatabaseAZ1: $[taskcat_gensingleaz_1] # Specific AZ for primary DB + DatabaseAZ2: $[taskcat_gensingleaz_2] # Specific AZ for standby DB ``` + +## Best Practices + +1. **Use Consistent Naming**: Choose descriptive parameter names that clearly indicate their purpose. + +2. **Password Security**: Always use `$[taskcat_genpass_XS]` for production passwords to include special characters. + +3. **Parameter Reuse**: Use `$[taskcat_getval_ParameterName]` for password confirmation fields to ensure consistency. + +4. **Region Awareness**: Use `$[taskcat_current_region]` when you need region-specific logic in your templates. + +5. **AZ Planning**: Use `$[taskcat_genaz_N]` for resources that need multiple AZs, and `$[taskcat_gensingleaz_N]` for resources that need specific AZ placement. + +6. **External Dependencies**: Use SSM and Secrets Manager pseudo-parameters for values that change between environments or contain sensitive data. + +## Troubleshooting + +### Common Issues + +- **Git Branch Parameter**: `$[taskcat_git_branch]` requires the project to be in a Git repository +- **AZ Availability**: Some regions may not have enough AZs for high numbers (e.g., `$[taskcat_genaz_5]`) +- **SSM Permissions**: Ensure taskcat has permissions to read from Parameter Store and Secrets Manager +- **Parameter Dependencies**: When using `$[taskcat_getval_X]`, ensure the referenced parameter is defined in the same test + +### Error Messages +- `"Project root is not a git repository"` - Use Git or avoid `$[taskcat_git_branch]` +- `"Not enough availability zones"` - Reduce the number in `$[taskcat_genaz_N]` or choose a different region +- `"Parameter not found"` - Check SSM parameter paths and permissions diff --git a/gendocs.sh b/gendocs.sh new file mode 100755 index 000000000..4b28ea93c --- /dev/null +++ b/gendocs.sh @@ -0,0 +1,1047 @@ +#!/bin/bash +# +# taskcat Documentation Generator Script +# +# This script automates the generation of taskcat documentation using MkDocs with +# Material theme. It provides options for local preview, building, and deployment. +# +# Usage: +# ./gendocs.sh [OPTIONS] +# +# Options: +# --preview, -p Start local development server for preview +# --build, -b Build documentation for production +# --deploy, -d Deploy to GitHub Pages +# --clean, -c Clean build artifacts +# --install, -i Install documentation dependencies +# --help, -h Show this help message +# +# Examples: +# ./gendocs.sh --preview # Preview locally at http://localhost:8000 +# ./gendocs.sh --build # Build static site to ./site/ +# ./gendocs.sh --deploy # Deploy to GitHub Pages +# + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$SCRIPT_DIR" + +# Documentation configuration +DOCS_DIR="$PROJECT_ROOT/docs" +SITE_DIR="$PROJECT_ROOT/site" +MKDOCS_CONFIG="$PROJECT_ROOT/mkdocs.yml" + +# Function to print colored output +print_status() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Function to check if command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Function to check Python and pip +check_python() { + if ! command_exists python3; then + print_error "Python 3 is required but not installed" + exit 1 + fi + + if ! command_exists pip3; then + print_error "pip3 is required but not installed" + exit 1 + fi + + print_status "Python $(python3 --version) detected" +} + +# Function to install documentation dependencies +install_deps() { + print_status "Installing documentation dependencies..." + + # Create requirements file for docs with enhanced packages + cat > "$PROJECT_ROOT/docs-requirements.txt" << EOF +mkdocs>=1.5.0 +mkdocs-material>=9.4.0 +mkdocstrings[python]>=0.23.0 +mkdocs-gen-files>=0.5.0 +mkdocs-literate-nav>=0.6.0 +mkdocs-section-index>=0.3.0 +mkdocs-minify-plugin>=0.7.0 +mkdocs-git-revision-date-localized-plugin>=1.2.0 +pymdown-extensions>=10.0.0 +pillow>=10.0.0 +cairosvg>=2.7.0 +EOF + + pip3 install -r "$PROJECT_ROOT/docs-requirements.txt" + print_success "Documentation dependencies installed" +} + +# Function to create MkDocs configuration +create_mkdocs_config() { + print_status "Creating MkDocs configuration..." + + cat > "$MKDOCS_CONFIG" << 'EOF' +site_name: taskcat Documentation +site_description: AWS CloudFormation Template Testing Tool +site_author: taskcat Team +site_url: https://aws-ia.github.io/taskcat/ + +repo_name: aws-ia/taskcat +repo_url: https://github.com/aws-ia/taskcat +edit_uri: edit/main/docs/ + +theme: + name: material + palette: + scheme: slate + features: + - content.code.copy + - content.code.annotate + - navigation.instant + - navigation.tracking + - navigation.sections + - navigation.indexes + - navigation.top + - search.highlight + - search.share + - search.suggest + - toc.follow + - toc.integrate + icon: + repo: fontawesome/brands/github + logo: assets/images/tcat.png + favicon: assets/images/tcat.png + font: + text: Roboto + code: Roboto Mono + +extra_css: + - https://cdn.jsdelivr.net/npm/bootswatch@5.2.3/dist/cyborg/bootstrap.min.css + - assets/css/cyborg-theme.css + +extra_javascript: + - assets/js/cyborg-theme.js + +plugins: + - search: + lang: en + - gen-files: + scripts: + - docs/gen_ref_pages.py + - literate-nav: + nav_file: SUMMARY.md + - section-index + - mkdocstrings: + handlers: + python: + options: + docstring_style: google + show_source: true + show_root_heading: true + show_root_toc_entry: false + merge_init_into_class: true + separate_signature: true + show_signature_annotations: true + +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - toc: + permalink: true + - pymdownx.arithmatex: + generic: true + - pymdownx.betterem: + smart_enable: all + - pymdownx.caret + - pymdownx.details + - pymdownx.emoji: + emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:materialx.emoji.twemoji + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.keys + - pymdownx.magiclink: + repo_url_shorthand: true + user: aws-ia + repo: taskcat + - pymdownx.mark + - pymdownx.smartsymbols + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.tilde + +nav: + - Home: index.md + - Getting Started: + - Installation: installation.md + - Quick Start: quickstart.md + - Configuration: configuration.md + - User Guide: + - Template Testing: user-guide/template-testing.md + - Multi-Region Testing: user-guide/multi-region.md + - Dynamic Values: user-guide/dynamic-values.md + - Parameter Overrides: user-guide/parameter-overrides.md + - Pseudo Parameters: user-guide/pseudo-parameters.md + - Examples: + - Basic Usage: examples/basic.md + - Advanced Scenarios: examples/advanced.md + - API Reference: reference/ + - Schema Reference: schema/ + - Support: + - Troubleshooting: support/troubleshooting.md + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/aws-ia/taskcat + - icon: fontawesome/brands/python + link: https://pypi.org/project/taskcat/ + +copyright: Copyright © 2023 Amazon Web Services +EOF + + print_success "MkDocs configuration created" +} + +# Function to create documentation structure +create_docs_structure() { + print_status "Creating comprehensive documentation structure..." + + # Create all necessary directories + mkdir -p "$DOCS_DIR"/{getting-started,user-guide,examples,support,overrides} + mkdir -p "$DOCS_DIR/assets"/{images,css,js} + + # Download taskcat logo if not present + if [ ! -f "$DOCS_DIR/assets/images/tcat.png" ]; then + print_status "Downloading taskcat logo..." + curl -o "$DOCS_DIR/assets/images/tcat.png" https://raw.githubusercontent.com/aws-ia/taskcat/main/assets/docs/images/tcat.png + if [ $? -eq 0 ]; then + print_success "taskcat logo downloaded successfully" + else + print_warning "Failed to download taskcat logo, using placeholder" + fi + fi + + # Copy existing documentation files if they exist + if [ -f "$PROJECT_ROOT/docs/usage/GENERAL_USAGE.md" ]; then + print_status "Preserving existing usage documentation..." + # Usage docs are already in place + fi + + if [ -f "$PROJECT_ROOT/docs/usage/PSUEDO_PARAMETERS.md" ]; then + print_status "Preserving enhanced pseudo-parameters documentation..." + # Already updated with comprehensive content + fi + + if [ -f "$PROJECT_ROOT/docs/usage/PARAMETER_OVERRIDES.md" ]; then + print_status "Preserving parameter overrides documentation..." + # Already in place + fi + + # Create enhanced installation guide + cat > "$DOCS_DIR/getting-started/installation.md" << 'EOF' +# Installation Guide + +Get taskcat installed and running on your system with our comprehensive installation guide. + +## Prerequisites + +Before installing taskcat, ensure you have: + +- **Python 3.8+**: taskcat requires Python 3.8 or higher +- **AWS CLI**: Configured with appropriate credentials +- **Git**: For cloning repositories and version control +- **Sufficient AWS Permissions**: See [Required Permissions](#required-permissions) + +## Installation Methods + +### Method 1: PyPI (Recommended) + +The easiest way to install taskcat is via PyPI: + +```bash +pip install taskcat +``` + +For the latest development version: + +```bash +pip install --upgrade taskcat +``` + +### Method 2: From Source + +For development or the latest features: + +```bash +git clone https://github.com/aws-ia/taskcat.git +cd taskcat +pip install -e . +``` + +### Method 3: Docker + +Use our pre-built Docker images: + +```bash +docker pull public.ecr.aws/aws-ia/taskcat:latest +docker run -it --rm -v $(pwd):/workspace taskcat --help +``` + +## Verification + +Verify your installation: + +```bash +taskcat --version +taskcat --help +``` + +## AWS Configuration + +### Configure AWS Credentials + +taskcat uses AWS credentials from your environment. Configure using: + +#### AWS CLI +```bash +aws configure +``` + +#### Environment Variables +```bash +export AWS_ACCESS_KEY_ID=your-access-key +export AWS_SECRET_ACCESS_KEY=your-secret-key +export AWS_DEFAULT_REGION=us-east-1 +``` + +#### IAM Roles (Recommended for EC2/Lambda) +taskcat automatically uses IAM roles when running on AWS services. + +### Required Permissions + +taskcat requires the following AWS permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "cloudformation:*", + "s3:*", + "iam:*", + "ec2:Describe*", + "ssm:GetParameter*", + "secretsmanager:GetSecretValue" + ], + "Resource": "*" + } + ] +} +``` + +## Next Steps + +Once installed, continue to the [Quick Start Guide](quickstart.md) to run your first test. + +## Troubleshooting + +### Common Issues + +**Python version too old:** +```bash +python3 --version # Should be 3.8+ +``` + +**Permission denied:** +```bash +pip install --user taskcat +``` + +**AWS credentials not found:** +```bash +aws configure list +``` + +For more help, see our [Troubleshooting Guide](../support/troubleshooting.md). +EOF + + # Create quick start guide + cat > "$DOCS_DIR/getting-started/quickstart.md" << 'EOF' +# Quick Start Guide + +Get up and running with taskcat in just a few minutes! This guide will walk you through creating and running your first taskcat test. + +## Step 1: Create a Simple Template + +First, let's create a basic CloudFormation template to test: + +```yaml +# templates/simple-s3.yaml +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Simple S3 bucket for taskcat testing' + +Parameters: + BucketName: + Type: String + Description: Name for the S3 bucket + Default: my-test-bucket + +Resources: + TestBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub "${BucketName}-${AWS::Region}-${AWS::AccountId}" + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + +Outputs: + BucketName: + Description: Name of the created bucket + Value: !Ref TestBucket + Export: + Name: !Sub "${AWS::StackName}-BucketName" +``` + +## Step 2: Create taskcat Configuration + +Create a taskcat configuration file: + +```yaml +# .taskcat.yml +project: + name: my-first-taskcat-test + regions: + - us-east-1 + - us-west-2 + +tests: + simple-test: + template: templates/simple-s3.yaml + parameters: + BucketName: $[taskcat_random-string] +``` + +## Step 3: Run Your First Test + +Execute the test: + +```bash +taskcat test run +``` + +taskcat will: +1. 🚀 Deploy your template in specified regions +2. ✅ Validate the deployment +3. 📊 Generate a detailed report +4. 🧹 Clean up resources + +## Step 4: View Results + +Check the results in the `taskcat_outputs` directory: + +```bash +ls taskcat_outputs/ +# index.html - Main report +# logs/ - Detailed logs +# templates/ - Processed templates +``` + +Open `taskcat_outputs/index.html` in your browser to see the visual report. + +## What Just Happened? + +taskcat performed these actions: + +1. **Template Processing**: Replaced pseudo-parameters with actual values +2. **Multi-Region Deployment**: Created CloudFormation stacks in us-east-1 and us-west-2 +3. **Validation**: Verified successful deployment and resource creation +4. **Reporting**: Generated comprehensive HTML and JSON reports +5. **Cleanup**: Automatically deleted test resources + +## Next Steps + +Now that you've run your first test, explore: + +- [Configuration Guide](configuration.md) - Advanced configuration options +- [Pseudo-Parameters](../usage/PSUEDO_PARAMETERS.md) - Dynamic parameter generation +- [Examples](../examples/) - Real-world usage scenarios + +## Common Next Actions + +### Test Multiple Templates +```yaml +tests: + test1: + template: templates/vpc.yaml + test2: + template: templates/ec2.yaml + parameters: + InstanceType: t3.micro +``` + +### Add Parameter Overrides +```yaml +tests: + production-test: + template: templates/app.yaml + parameters: + Environment: prod + InstanceType: m5.large +``` + +### Customize Regions +```yaml +project: + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 +``` + +Congratulations! You've successfully run your first taskcat test. 🎉 +EOF + + # Create configuration guide + cat > "$DOCS_DIR/getting-started/configuration.md" << 'EOF' +# Configuration Guide + +Learn how to configure taskcat for your specific testing needs with comprehensive configuration options. + +## Configuration File Structure + +taskcat uses YAML configuration files (`.taskcat.yml`) with this structure: + +```yaml +project: + name: string # Project name + regions: [list] # AWS regions to test + s3_bucket: string # Optional: Custom S3 bucket + s3_key_prefix: string # Optional: S3 key prefix + +tests: + test-name: # Test identifier + template: string # Path to CloudFormation template + parameters: {} # Parameter overrides + regions: [list] # Optional: Test-specific regions + +global: + parameters: {} # Global parameter overrides +``` + +## Project Configuration + +### Basic Project Settings + +```yaml +project: + name: my-cloudformation-project + regions: + - us-east-1 + - us-west-2 + - eu-west-1 +``` + +### Advanced Project Settings + +```yaml +project: + name: enterprise-infrastructure + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 + s3_bucket: my-custom-taskcat-bucket + s3_key_prefix: testing/templates/ + tags: + Environment: Testing + Project: taskcat + Owner: DevOps-Team +``` + +## Test Configuration + +### Single Test + +```yaml +tests: + basic-test: + template: templates/main.yaml + parameters: + InstanceType: t3.micro + Environment: test +``` + +### Multiple Tests + +```yaml +tests: + small-deployment: + template: templates/small.yaml + parameters: + InstanceType: t3.micro + + large-deployment: + template: templates/large.yaml + parameters: + InstanceType: m5.xlarge + + multi-az-test: + template: templates/multi-az.yaml + regions: + - us-east-1 + - us-west-2 + parameters: + AvailabilityZones: $[taskcat_genaz_3] +``` + +## Parameter Management + +### Global Parameters + +Parameters that apply to all tests: + +```yaml +global: + parameters: + KeyPairName: my-keypair + VpcCidr: 10.0.0.0/16 + Environment: testing + +tests: + test1: + template: templates/app.yaml + # Inherits global parameters + test2: + template: templates/db.yaml + # Also inherits global parameters +``` + +### Test-Specific Parameters + +Override global parameters for specific tests: + +```yaml +global: + parameters: + Environment: testing + InstanceType: t3.micro + +tests: + production-test: + template: templates/app.yaml + parameters: + Environment: production # Overrides global + InstanceType: m5.large # Overrides global +``` + +### Pseudo-Parameters + +Use dynamic parameters for flexible testing: + +```yaml +tests: + dynamic-test: + template: templates/app.yaml + parameters: + # Generate random values + DatabasePassword: $[taskcat_genpass_16S] + S3Bucket: $[taskcat_autobucket] + + # Use current context + Region: $[taskcat_current_region] + ProjectName: $[taskcat_project_name] + + # Generate availability zones + AvailabilityZones: $[taskcat_genaz_2] + + # Reference other parameters + PasswordConfirm: $[taskcat_getval_DatabasePassword] +``` + +## Region Configuration + +### Project-Level Regions + +All tests use these regions by default: + +```yaml +project: + regions: + - us-east-1 + - us-west-2 + - eu-west-1 +``` + +### Test-Specific Regions + +Override regions for specific tests: + +```yaml +project: + regions: + - us-east-1 + - us-west-2 + +tests: + global-test: + template: templates/global.yaml + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 + + us-only-test: + template: templates/us-specific.yaml + regions: + - us-east-1 + - us-west-2 +``` + +## Advanced Configuration + +### Custom S3 Configuration + +```yaml +project: + name: my-project + s3_bucket: my-custom-bucket-${AWS::Region} + s3_key_prefix: taskcat-tests/ + s3_object_acl: private +``` + +### Authentication Configuration + +```yaml +project: + auth: + us-east-1: profile1 + us-west-2: profile2 + default: default-profile +``` + +### Template Processing + +```yaml +project: + template: + transforms: + - AWS::Serverless-2016-10-31 + capabilities: + - CAPABILITY_IAM + - CAPABILITY_NAMED_IAM +``` + +## Configuration Examples + +### Microservices Architecture + +```yaml +project: + name: microservices-platform + regions: + - us-east-1 + - us-west-2 + - eu-west-1 + +global: + parameters: + Environment: testing + VpcCidr: 10.0.0.0/16 + +tests: + vpc-infrastructure: + template: templates/vpc.yaml + + application-tier: + template: templates/app-tier.yaml + parameters: + InstanceType: t3.medium + + database-tier: + template: templates/db-tier.yaml + parameters: + DBInstanceClass: db.t3.micro + + monitoring: + template: templates/monitoring.yaml + regions: + - us-east-1 # Only deploy monitoring in primary region +``` + +### Multi-Environment Testing + +```yaml +project: + name: multi-env-app + regions: + - us-east-1 + - us-west-2 + +tests: + development: + template: templates/app.yaml + parameters: + Environment: dev + InstanceType: t3.micro + + staging: + template: templates/app.yaml + parameters: + Environment: staging + InstanceType: t3.small + + production: + template: templates/app.yaml + parameters: + Environment: prod + InstanceType: m5.large +``` + +## Best Practices + +### 1. Use Meaningful Names +```yaml +tests: + vpc-with-public-subnets: # ✅ Descriptive + template: templates/vpc.yaml + + test1: # ❌ Not descriptive + template: templates/vpc.yaml +``` + +### 2. Organize Parameters +```yaml +global: + parameters: + # Common across all tests + Environment: testing + Owner: devops-team + +tests: + web-tier: + parameters: + # Specific to this test + InstanceType: t3.medium + MinSize: 2 + MaxSize: 10 +``` + +### 3. Use Pseudo-Parameters +```yaml +parameters: + # ✅ Dynamic and flexible + DatabasePassword: $[taskcat_genpass_16S] + S3Bucket: $[taskcat_autobucket] + + # ❌ Static and potentially conflicting + DatabasePassword: hardcoded-password + S3Bucket: my-static-bucket-name +``` + +## Validation + +Validate your configuration: + +```bash +# Check configuration syntax +taskcat test run --dry-run + +# Lint CloudFormation templates +taskcat lint + +# Generate configuration schema +taskcat schema +``` + +## Next Steps + +- [Pseudo-Parameters Guide](../usage/PSUEDO_PARAMETERS.md) +- [Parameter Overrides](../usage/PARAMETER_OVERRIDES.md) +- [Advanced Examples](../examples/advanced.md) +EOF + + print_success "Comprehensive documentation structure created" +} + +# Function to build documentation +build_docs() { + print_status "Building documentation..." + + if [ ! -f "$MKDOCS_CONFIG" ]; then + print_error "MkDocs configuration not found. Run with --install first." + exit 1 + fi + + cd "$PROJECT_ROOT" + mkdocs build --clean + + print_success "Documentation built successfully" + print_status "Static site available in: $SITE_DIR" +} + +# Function to serve documentation locally +serve_docs() { + print_status "Starting local documentation server..." + + if [ ! -f "$MKDOCS_CONFIG" ]; then + print_error "MkDocs configuration not found. Run with --install first." + exit 1 + fi + + cd "$PROJECT_ROOT" + print_success "Documentation server starting..." + print_status "Open your browser to: http://localhost:8000" + print_status "Press Ctrl+C to stop the server" + + mkdocs serve --dev-addr=localhost:8000 +} + +# Function to deploy to GitHub Pages +deploy_docs() { + print_status "Deploying documentation to GitHub Pages..." + + if [ ! -f "$MKDOCS_CONFIG" ]; then + print_error "MkDocs configuration not found. Run with --install first." + exit 1 + fi + + cd "$PROJECT_ROOT" + mkdocs gh-deploy --clean + + print_success "Documentation deployed to GitHub Pages" +} + +# Function to clean build artifacts +clean_docs() { + print_status "Cleaning documentation build artifacts..." + + if [ -d "$SITE_DIR" ]; then + rm -rf "$SITE_DIR" + print_status "Removed $SITE_DIR" + fi + + if [ -f "$PROJECT_ROOT/docs-requirements.txt" ]; then + rm "$PROJECT_ROOT/docs-requirements.txt" + print_status "Removed docs-requirements.txt" + fi + + print_success "Clean completed" +} + +# Function to show help +show_help() { + cat << 'EOF' +taskcat Documentation Generator + +Usage: ./gendocs.sh [OPTIONS] + +Options: + --preview, -p Start local development server for preview + --build, -b Build documentation for production + --deploy, -d Deploy to GitHub Pages + --clean, -c Clean build artifacts + --install, -i Install documentation dependencies and setup + --help, -h Show this help message + +Examples: + ./gendocs.sh --install # First time setup + ./gendocs.sh --preview # Preview locally at http://localhost:8000 + ./gendocs.sh --build # Build static site to ./site/ + ./gendocs.sh --deploy # Deploy to GitHub Pages + ./gendocs.sh --clean # Clean build artifacts + +Workflow: + 1. Run --install to set up dependencies and configuration + 2. Run --preview to develop and test documentation locally + 3. Run --build to create production build + 4. Run --deploy to publish to GitHub Pages +EOF +} + +# Main script logic +main() { + case "${1:-}" in + --preview|-p) + check_python + serve_docs + ;; + --build|-b) + check_python + build_docs + ;; + --deploy|-d) + check_python + deploy_docs + ;; + --clean|-c) + clean_docs + ;; + --install|-i) + check_python + install_deps + create_mkdocs_config + create_docs_structure + print_success "Documentation setup completed!" + print_status "Next steps:" + print_status " 1. Run './gendocs.sh --preview' to preview locally" + print_status " 2. Edit documentation files in the docs/ directory" + print_status " 3. Run './gendocs.sh --deploy' to publish to GitHub Pages" + ;; + --help|-h|"") + show_help + ;; + *) + print_error "Unknown option: $1" + show_help + exit 1 + ;; + esac +} + +# Run main function with all arguments +main "$@" diff --git a/generate_config_docs.py b/generate_config_docs.py deleted file mode 100644 index 7f0387bbc..000000000 --- a/generate_config_docs.py +++ /dev/null @@ -1,78 +0,0 @@ -# flake8: noqa -import json - -if __name__ == "__main__": - - schema = json.load(open("./taskcat/cfg/config_schema.json", "r")) - - def resolve_ref(props): - if "$ref" in props: - ref = props["$ref"].split("/")[-1] - del props["$ref"] - props.update(schema["definitions"][ref]) - return props - - for k, v in schema["properties"].items(): - item_str = f"* `{k}` " - v = resolve_ref(v) - item_str += f"*type:* `{v['type']}` " - if "description" in v: - item_str += f'{v["description"]}' - print(item_str) - if "properties" in v: - for ik, iv in v["properties"].items(): - item_str = f" * `{ik}` " - iv = resolve_ref(iv) - item_str += f"*type:* `{iv['type']}` " - if "description" in iv: - item_str += f'{iv["description"]}' - print(item_str) - if iv["type"] == "object": - if "properties" in iv: - for iik, iiv in iv["properties"].items(): - item_str = f" * `{iik}` " - iiv = resolve_ref(iiv) - item_str += f"*type:* `{iiv['type']}` " - if "description" in iiv: - item_str += f'{iiv["description"]}' - elif "additionalProperties" in iv: - name = ik[:-1] if ik.endswith("s") else ik - item_str = f" * `<{name.upper()}_NAME>` " - props = resolve_ref(iv["additionalProperties"]) - item_str += f"*type:* `{iv['type']}` " - if "description" in props: - item_str += f'{props["description"]}' - print(item_str) - elif "additionalProperties" in v: - name = k[:-1] if k.endswith("s") else k - item_str = f" * `<{name.upper()}_NAME>` " - props = resolve_ref(v["additionalProperties"]) - item_str += f"*type:* `{v['type']}` " - if "description" in props: - item_str += f'{props["description"]}' - if "properties" in props: - for ik, iv in props["properties"].items(): - item_str = f" * `{ik}` " - iv = resolve_ref(iv) - item_str += f"*type:* `{iv['type']}` " - if "description" in iv: - item_str += f'{iv["description"]}' - print(item_str) - if iv["type"] == "object": - if "properties" in iv: - for iik, iiv in iv["properties"].items(): - item_str = f" * `{iik}` " - iiv = resolve_ref(iiv) - item_str += f"*type:* `{iiv['type']}` " - if "description" in iiv: - item_str += f'{iiv["description"]}' - elif "additionalProperties" in iv: - name = ik[:-1] if ik.endswith("s") else ik - item_str = f" * `<{name.upper()}_NAME>` " - iprops = resolve_ref(iv["additionalProperties"]) - item_str += f"*type:* `{iv['type']}` " - if "description" in iprops: - item_str += f'{iprops["description"]}' - print(item_str) - else: - print(v) diff --git a/generate_schema.py b/generate_schema.py index 4f5d92308..861c6ed9b 100755 --- a/generate_schema.py +++ b/generate_schema.py @@ -1,11 +1,46 @@ #!/usr/bin/env python +""" +TaskCat Configuration Schema Generator + +This script generates a JSON schema file for TaskCat configuration validation. +The schema is derived from the BaseConfig dataclass and is used to validate +TaskCat configuration files (.taskcat.yml) to ensure they conform to the +expected structure and contain valid values. + +The generated schema file is used by: +- IDE extensions for configuration file validation and auto-completion +- TaskCat itself for runtime configuration validation +- Documentation generation tools +- CI/CD pipelines for configuration validation + +Usage: + python generate_schema.py + +Output: + Creates/updates ./taskcat/cfg/config_schema.json with the current schema +""" + import json from taskcat._dataclasses import BaseConfig if __name__ == "__main__": + # Generate JSON schema from the BaseConfig dataclass + # This uses the dataclasses-jsonschema library to automatically + # create a comprehensive JSON schema based on the dataclass definitions schema = BaseConfig.json_schema() + + # Write the schema to the configuration directory + # The schema file is used for validation and IDE support with open("./taskcat/cfg/config_schema.json", "w") as f: - f.write(json.dumps(schema, sort_keys=True, indent=4, separators=(",", ": "))) - + # Format the JSON with consistent indentation and sorting + # This ensures the schema file is human-readable and diff-friendly + f.write(json.dumps( + schema, + sort_keys=True, # Sort keys alphabetically for consistency + indent=4, # Use 4-space indentation for readability + separators=(",", ": ") # Clean separator formatting + )) + + # Add final newline for POSIX compliance f.write("\n") diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..794ca7f64 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,120 @@ +site_name: taskcat +site_description: AWS CloudFormation Template Testing Tool +site_author: AWS Solutions Architecture Team +site_url: https://aws-ia.github.io/taskcat/ + +repo_name: aws-ia/taskcat +repo_url: https://github.com/aws-ia/taskcat +edit_uri: edit/main/docs/ + +theme: + name: material + palette: + scheme: slate + features: + - content.code.copy + - content.code.annotate + - navigation.instant + - navigation.tracking + - navigation.sections + - navigation.indexes + - navigation.top + - search.highlight + - search.share + - search.suggest + - toc.follow + - toc.integrate + icon: + repo: fontawesome/brands/github + logo: assets/images/tcat.png + favicon: assets/images/tcat.png + font: + text: Roboto + code: Roboto Mono + +extra_css: + - https://cdn.jsdelivr.net/npm/bootswatch@5.2.3/dist/cyborg/bootstrap.min.css + - assets/css/cyborg-theme.css + +extra_javascript: + - assets/js/cyborg-theme.js + +plugins: + - search: + lang: en + - gen-files: + scripts: + - docs/gen_ref_pages.py + - literate-nav: + nav_file: SUMMARY.md + - section-index + - mkdocstrings: + handlers: + python: + options: + docstring_style: google + show_source: true + show_root_heading: true + merge_init_into_class: true + +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - toc: + permalink: true + - tables + - pymdownx.betterem: + smart_enable: all + - pymdownx.caret + - pymdownx.details + - pymdownx.emoji: + emoji_generator: !!python/name:material.extensions.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + use_pygments: true + - pymdownx.inlinehilite + - pymdownx.keys + - pymdownx.mark + - pymdownx.smartsymbols + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true + combine_header_slug: true + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.tilde + +nav: + - Home: index.md + - Installation: installation.md + - Quick Start: quickstart.md + - Configuration: configuration.md + - Dynamic Values: dynamic-values.md + - Parameter Overrides: parameter-overrides.md + - Examples: examples.md + - Schema Reference: schema.md + - API Reference: api.md + - Troubleshooting: troubleshooting.md + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/aws-ia/taskcat + name: GitHub Repository + - icon: fontawesome/brands/python + link: https://pypi.org/project/taskcat/ + name: PyPI Package + generator: false + +copyright: Copyright © 2023 Amazon Web Services, Inc. diff --git a/pyproject.toml b/pyproject.toml index 5a3e8d20c..fc2c124f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,47 +29,3 @@ python = ">=3.8.0,<4.0" [tool.poetry.scripts] taskcat = "taskcat._cli:main" - -[tool.portray.mkdocs.theme] -favicon = "https://raw.githubusercontent.com/aws-ia/taskcat/main/assets/docs/images/tcat.png" -logo = "https://raw.githubusercontent.com/aws-ia/taskcat/main/assets/docs/images/tcat.png" -name = "material" -palette = {primary = "black", accent = "orange"} - - - -[tool.portray.mkdocs] -extra_css = ['docs/custom.css'] - -[[tool.portray.mkdocs.nav]] -Home = "README.md" - -[[tool.portray.mkdocs.nav]] -Installation = "docs/INSTALLATION.md" - - -[[tool.portray.mkdocs.nav]] - [[tool.portray.mkdocs.nav.Usage]] - "General Usage" = "docs/usage/GENERAL_USAGE.md" - - [[tool.portray.mkdocs.nav.Usage]] - "Parameter Overrides" = "docs/usage/PARAMETER_OVERRIDES.md" - - [[tool.portray.mkdocs.nav.Usage]] - "Psuedo Parameters" = "docs/usage/PSUEDO_PARAMETERS.md" - -[[tool.portray.mkdocs.nav]] -"Configuration Schema" = "docs/schema/taskcat_schema.md" - -[[tool.portray.mkdocs.nav]] - [[tool.portray.mkdocs.nav.Administrative]] - "Code of Conduct" = "docs/administrative/CODE_OF_CONDUCT.md" - - [[tool.portray.mkdocs.nav.Administrative]] - "Contributing" = "docs/administrative/CONTRIBUTING.md" - -# [[tool.portray.mkdocs.nav]] -# Home = "README.md" -# -# [[tool.portray.mkdocs.nav]] -# Usage = "docs/usage/" diff --git a/taskcat/_cli.py b/taskcat/_cli.py index 49c9ad781..61ffdf0d4 100644 --- a/taskcat/_cli.py +++ b/taskcat/_cli.py @@ -1,3 +1,14 @@ +""" +TaskCat CLI Entry Point Module + +This module provides the main command-line interface for TaskCat, an AWS CloudFormation +template testing tool. It handles CLI argument parsing, logging setup, version checking, +and error handling for the entire application. + +The module serves as the primary entry point when TaskCat is invoked from the command line, +coordinating between various CLI modules and providing a consistent user experience. +""" + import importlib.metadata import signal import sys @@ -11,14 +22,20 @@ from . import _cli_modules +# Initialize logger with ERROR level by default LOG = init_taskcat_cli_logger(loglevel="ERROR") + +# ASCII art banner displayed when TaskCat starts BANNER = ( " _ _ _ \n| |_ __ _ ___| | _____ __ _| |_ \n| __/ _" "` / __| |/ / __/ _` | __|\n| || (_| \\__ \\ < (_| (_| | |_ \n \\__\\__,_|" "___/_|\\_\\___\\__,_|\\__|\n \n" ) +# Application name used throughout the CLI NAME = "taskcat" + +# Application description shown in help text DESCRIPTION = ( "taskcat is a tool that tests AWS CloudFormation templates. It deploys " "your AWS CloudFormation template in multiple AWS Regions and " @@ -30,24 +47,65 @@ def main(cli_core_class=CliCore, exit_func=exit_with_code): + """ + Main entry point for the TaskCat CLI application. + + This function orchestrates the entire CLI workflow including: + - Signal handling setup for graceful interruption + - Logging configuration + - Command-line argument parsing + - CLI module initialization and execution + - Error handling and reporting + + Args: + cli_core_class (class, optional): CLI core class to use for parsing and execution. + Defaults to CliCore. Used for dependency injection + in testing. + exit_func (callable, optional): Function to call for program exit. + Defaults to exit_with_code. Used for testing. + + Raises: + TaskCatException: For known TaskCat-specific errors + Exception: For unexpected errors during execution + """ + # Set up signal handler for graceful interruption (Ctrl+C) signal.signal(signal.SIGINT, _sigint_handler) + + # Configure logging based on command-line arguments log_level = _setup_logging(sys.argv) + + # Get command-line arguments, default to help if none provided args = sys.argv[1:] if not args: args.append("-h") + try: + # Display welcome banner and version information _welcome() + + # Get the currently installed version of TaskCat version = get_installed_version() + + # Initialize the CLI core with modules and configuration cli = cli_core_class(NAME, _cli_modules, DESCRIPTION, version, GLOBAL_ARGS.ARGS) + + # Parse the command-line arguments cli.parse(args) + + # Extract and set the AWS profile if specified _default_profile = cli.parsed_args.__dict__.get("_profile") if _default_profile: GLOBAL_ARGS.profile = _default_profile + + # Execute the parsed command cli.run() + except TaskCatException as e: + # Handle known TaskCat exceptions with appropriate logging LOG.error(str(e), exc_info=_print_tracebacks(log_level)) exit_func(1) except Exception as e: # pylint: disable=broad-except + # Handle unexpected exceptions with full error details LOG.error( "%s %s", e.__class__.__name__, str(e), exc_info=_print_tracebacks(log_level) ) @@ -55,24 +113,60 @@ def main(cli_core_class=CliCore, exit_func=exit_with_code): def _setup_logging(args, exit_func=exit_with_code): + """ + Configure logging level based on command-line arguments. + + Args: + args (list): Command-line arguments to parse for log level + exit_func (callable, optional): Function to call on exit. Defaults to exit_with_code. + + Returns: + str: The configured log level (e.g., 'DEBUG', 'INFO', 'ERROR') + """ log_level = _get_log_level(args, exit_func=exit_func) LOG.setLevel(log_level) return log_level def _print_tracebacks(log_level): + """ + Determine whether to print full tracebacks based on log level. + + Args: + log_level (str): Current logging level + + Returns: + bool: True if tracebacks should be printed (DEBUG mode), False otherwise + """ return log_level == "DEBUG" def _print_upgrade_msg(new_version, version): + """ + Display upgrade notification message to the user. + + Shows available upgrade options for both pip and Docker installations + when a newer version of TaskCat is available. + + Args: + new_version (str): The latest available version + version (str): The currently installed version + """ + # Display current version LOG.info(f"version {version}\n", extra={"nametag": ""}) + + # Warn about newer version availability LOG.warning("A newer version of %s is available (%s)", NAME, new_version) + + # Show pip upgrade command LOG.info( "To upgrade pip version %s[ pip install --upgrade %s]%s", PrintMsg.highlight, NAME, PrintMsg.rst_color, ) + + # Show Docker upgrade command LOG.info( "To upgrade docker version %s[ docker pull %s/%s ]%s\n", PrintMsg.highlight, @@ -83,42 +177,105 @@ def _print_upgrade_msg(new_version, version): def check_for_update(): + """ + Check for available TaskCat updates and notify the user. + + Compares the currently installed version with the latest version available + on PyPI. Only checks for stable releases (non-dev versions) and handles + various error conditions gracefully. + + The function will: + - Skip update checks for development versions + - Display current version information + - Show upgrade instructions if a newer version is available + - Handle network errors and API failures gracefully + """ version = get_installed_version() + + # Skip update check for local development installations if version != "[local source] no pip module installed": + # Only check for updates on stable releases (not dev versions) if "dev" not in version: try: + # Fetch latest version from PyPI current_version = get_pip_version(f"https://pypi.org/pypi/{NAME}/json") + if version in current_version: + # Current version is up to date LOG.info("version %s" % version, extra={"nametag": ""}) else: + # Newer version available, show upgrade message _print_upgrade_msg(current_version, version) + except Exception: # pylint: disable=broad-except + # Handle network errors, API failures, etc. LOG.debug("Unexpected error", exc_info=True) LOG.warning("Unable to get version info!!, continuing") else: + # Running from local source (development mode) LOG.info("Using local source (development mode)\n") def _welcome(): + """ + Display the TaskCat welcome banner and perform version checking. + + Shows the ASCII art banner and checks for available updates. + Handles any errors during the welcome process gracefully to avoid + interrupting the main application flow. + """ + # Display the ASCII art banner LOG.info(f"{BANNER}\n", extra={"nametag": ""}) + try: + # Check for available updates check_for_update() except Exception: # pylint: disable=broad-except + # Don't let version checking errors interrupt the application LOG.debug("Unexpected error", exc_info=True) LOG.warning("Unable to get version info!!, continuing") def get_pip_version(url): """ - Given the url to PypI package info url returns the current live version + Retrieve the current version of a package from PyPI. + + Args: + url (str): The PyPI JSON API URL for the package + + Returns: + str: The latest version string from PyPI + + Raises: + requests.RequestException: If the HTTP request fails + KeyError: If the expected JSON structure is not found + ValueError: If the JSON response is malformed """ - return requests.get(url, timeout=5.0).json()["info"]["version"] + response = requests.get(url, timeout=5.0) + return response.json()["info"]["version"] def get_installed_version(): + """ + Get the currently installed version of TaskCat. + + Uses importlib.metadata to retrieve version information from the + installed package metadata. + + Returns: + str: The installed version string, or a special message for + development installations + """ return importlib.metadata.version(__package__ or __name__) def _sigint_handler(signum, frame): + """ + Handle SIGINT (Ctrl+C) signal for graceful shutdown. + + Args: + signum (int): The signal number that was received + frame: The current stack frame when the signal was received + """ LOG.debug(f"SIGNAL {signum} caught at {frame}") exit_with_code(1) diff --git a/taskcat/_common_utils.py b/taskcat/_common_utils.py index e6d524462..3424e7c7d 100644 --- a/taskcat/_common_utils.py +++ b/taskcat/_common_utils.py @@ -1,3 +1,20 @@ +""" +TaskCat Common Utilities Module + +This module provides a collection of utility functions and classes used throughout +the TaskCat application. It includes functions for: + +- AWS resource manipulation (S3, CloudFormation, SSM, Secrets Manager) +- String and data structure processing +- File system operations +- URL parsing and construction +- Configuration management +- Git submodule handling + +These utilities support the core TaskCat functionality by providing reusable +components for common operations across different modules. +""" + import collections.abc import logging import os @@ -16,89 +33,243 @@ from taskcat.exceptions import TaskCatException from taskcat.regions_to_partitions import REGIONS +# Initialize module logger LOG = logging.getLogger(__name__) +# Mapping of AWS partitions to their S3 domain suffixes S3_PARTITION_MAP = { - "aws": "amazonaws.com", - "aws-cn": "amazonaws.com.cn", - "aws-us-gov": "amazonaws.com", + "aws": "amazonaws.com", # Standard AWS partition + "aws-cn": "amazonaws.com.cn", # AWS China partition + "aws-us-gov": "amazonaws.com", # AWS GovCloud partition } +# Regular expressions for converting PascalCase to snake_case FIRST_CAP_RE = re.compile("(.)([A-Z][a-z]+)") ALL_CAP_RE = re.compile("([a-z0-9])([A-Z])") def region_from_stack_id(stack_id): + """ + Extract the AWS region from a CloudFormation stack ID. + + CloudFormation stack IDs follow the format: + arn:aws:cloudformation:region:account-id:stack/stack-name/unique-id + + Args: + stack_id (str): The CloudFormation stack ID/ARN + + Returns: + str: The AWS region code (e.g., 'us-east-1', 'eu-west-1') + + Example: + >>> stack_id = "arn:aws:cloudformation:us-east-1:123456789012:stack/my-stack/abc123" + >>> region_from_stack_id(stack_id) + 'us-east-1' + """ return stack_id.split(":")[3] def name_from_stack_id(stack_id): + """ + Extract the stack name from a CloudFormation stack ID. + + CloudFormation stack IDs follow the format: + arn:aws:cloudformation:region:account-id:stack/stack-name/unique-id + + Args: + stack_id (str): The CloudFormation stack ID/ARN + + Returns: + str: The CloudFormation stack name + + Example: + >>> stack_id = "arn:aws:cloudformation:us-east-1:123456789012:stack/my-stack/abc123" + >>> name_from_stack_id(stack_id) + 'my-stack' + """ return stack_id.split(":")[5].split("/")[1] def s3_url_maker(bucket, key, s3_client, autobucket=False): + """ + Generate the correct S3 URL for a given bucket and key. + + This function determines the appropriate S3 endpoint URL based on the bucket's + region and AWS partition. It handles different AWS partitions (standard, China, GovCloud) + and can optionally wait for bucket creation if autobucket is enabled. + + Args: + bucket (str): The S3 bucket name + key (str): The S3 object key/path + s3_client: Boto3 S3 client instance + autobucket (bool, optional): If True, retry on NoSuchBucket errors. + Defaults to False. + + Returns: + str: The complete S3 URL for the object + + Raises: + TaskCatException: If bucket region cannot be determined + ClientError: If bucket access is denied and region cannot be discovered + NoSuchBucket: If bucket doesn't exist and autobucket is False + + Example: + >>> url = s3_url_maker('my-bucket', 'path/to/file.txt', s3_client) + >>> print(url) + 'https://my-bucket.s3.us-west-2.amazonaws.com/path/to/file.txt' + """ retries = 10 + while True: try: try: + # Try to get bucket location directly response = s3_client.get_bucket_location(Bucket=bucket) location = response["LocationConstraint"] except ClientError as e: + # If access denied, try to discover region via HTTP headers if e.response["Error"]["Code"] != "AccessDenied": raise + + # Make a HEAD request to discover the bucket region resp = requests.get( f"https://{bucket}.s3.amazonaws.com/{key}", timeout=3 ) location = resp.headers.get("x-amz-bucket-region") + if not location: # pylint: disable=raise-missing-from raise TaskCatException( - f"failed to discover region for bucket {bucket}" + f"Failed to discover region for bucket {bucket}. " + f"Ensure the bucket exists and you have appropriate permissions." ) break + except s3_client.exceptions.NoSuchBucket: + # Handle case where bucket doesn't exist yet (for auto-created buckets) if not autobucket or retries < 1: raise retries -= 1 - sleep(5) + sleep(5) # Wait for bucket creation - # default case for us-east-1 which returns no location + # Default case for us-east-1 which returns None as LocationConstraint url = f"https://{bucket}.s3.us-east-1.amazonaws.com/{key}" + if location: + # Get the appropriate domain for the bucket's region/partition domain = get_s3_domain(location) url = f"https://{bucket}.s3.{location}.{domain}/{key}" + return url def get_s3_domain(region): + """ + Get the appropriate S3 domain suffix for a given AWS region. + + Different AWS partitions use different domain suffixes for S3 endpoints. + This function maps regions to their correct domain based on the partition. + + Args: + region (str): AWS region code (e.g., 'us-east-1', 'cn-north-1') + + Returns: + str: The S3 domain suffix for the region's partition + + Raises: + TaskCatException: If the region is not found in the partition mapping + + Example: + >>> get_s3_domain('us-east-1') + 'amazonaws.com' + >>> get_s3_domain('cn-north-1') + 'amazonaws.com.cn' + """ try: return S3_PARTITION_MAP[REGIONS[region]] except KeyError: # pylint: disable=raise-missing-from - raise TaskCatException(f"cannot find the S3 hostname for region {region}") + raise TaskCatException( + f"Cannot find the S3 hostname for region '{region}'. " + f"This region may not be supported or the region code may be invalid." + ) def s3_bucket_name_from_url(url): + """ + Extract the S3 bucket name from an S3 URL. + + Args: + url (str): S3 URL in format https://bucket-name.s3.region.amazonaws.com/key + + Returns: + str: The S3 bucket name + + Example: + >>> s3_bucket_name_from_url('https://my-bucket.s3.us-east-1.amazonaws.com/file.txt') + 'my-bucket' + """ return url.split("//")[1].split(".")[0] def s3_key_from_url(url): + """ + Extract the S3 object key from an S3 URL. + + Args: + url (str): S3 URL in format https://bucket-name.s3.region.amazonaws.com/key + + Returns: + str: The S3 object key/path + + Example: + >>> s3_key_from_url('https://my-bucket.s3.us-east-1.amazonaws.com/path/to/file.txt') + 'path/to/file.txt' + """ return "/".join(url.split("//")[1].split("/")[1:]) class CommonTools: + """ + Collection of common utility methods for TaskCat operations. + + This class provides static utility methods that are used across different + TaskCat modules for common operations like regex matching and string processing. + + Attributes: + stack_name (str): The CloudFormation stack name associated with this instance + """ + def __init__(self, stack_name): + """ + Initialize CommonTools with a stack name. + + Args: + stack_name (str): The CloudFormation stack name to associate with this instance + """ self.stack_name = stack_name @staticmethod def regxfind(re_object, data_line): """ - Returns the matching string. - - :param re_object: Regex object - :param data_line: String to be searched - - :return: Matching String if found, otherwise return 'Not-found' + Find and return the first regex match in a string. + + This method searches for a pattern in the provided string and returns + the matching substring. If no match is found, it returns 'Not-found'. + + Args: + re_object (re.Pattern): Compiled regular expression object + data_line (str): String to search for the pattern + + Returns: + str: The matching string if found, otherwise 'Not-found' + + Example: + >>> import re + >>> pattern = re.compile(r'sg-[a-f0-9]+') + >>> result = CommonTools.regxfind(pattern, 'SecurityGroup: sg-12345abc') + >>> print(result) + 'sg-12345abc' """ security_group = re_object.search(data_line) if security_group: @@ -107,46 +278,123 @@ def regxfind(re_object, data_line): def exit_with_code(code, msg=""): + """ + Exit the application with a specific exit code and optional message. + + This function provides a centralized way to exit the application with + proper logging and exit code handling. + + Args: + code (int): Exit code to return to the operating system + msg (str, optional): Error message to log before exiting. Defaults to empty string. + """ if msg: LOG.error(msg) sys.exit(code) def make_dir(path, ignore_exists=True): + """ + Create a directory and any necessary parent directories. + + Args: + path (str): Path to the directory to create + ignore_exists (bool, optional): If True, don't raise an error if directory + already exists. Defaults to True. + + Raises: + OSError: If directory creation fails or if ignore_exists is False and + directory already exists + """ path = os.path.abspath(path) + + # Skip creation if directory exists and ignore_exists is True if ignore_exists and os.path.isdir(path): return + + # Create directory and any necessary parent directories os.makedirs(path) def param_list_to_dict(original_keys): - # Setup a list index dictionary. - # - Used to give an Parameter => Index mapping for replacement. + """ + Convert a CloudFormation parameter list to a parameter index dictionary. + + CloudFormation parameters are often provided as a list of dictionaries with + 'ParameterKey' and 'ParameterValue' keys. This function creates an index + mapping parameter names to their positions in the list. + + Args: + original_keys (list): List of parameter dictionaries, each containing + 'ParameterKey' and 'ParameterValue' keys + + Returns: + dict: Dictionary mapping parameter names to their list indices + + Raises: + TaskCatException: If the input is not a list, if any parameter is not a dict, + or if required keys are missing + + Example: + >>> params = [ + ... {'ParameterKey': 'VpcId', 'ParameterValue': 'vpc-12345'}, + ... {'ParameterKey': 'SubnetId', 'ParameterValue': 'subnet-67890'} + ... ] + >>> param_list_to_dict(params) + {'VpcId': 0, 'SubnetId': 1} + """ + # Setup a list index dictionary for Parameter => Index mapping param_index = {} + + # Validate input is a list if not isinstance(original_keys, list): # pylint: disable=raise-missing-from raise TaskCatException( - 'Invalid parameter file, outermost json element must be a list ("[]")' + 'Invalid parameter file: outermost JSON element must be a list ("[]")' ) + + # Process each parameter in the list for idx, param_dict in enumerate(original_keys): + # Validate each parameter is a dictionary if not isinstance(param_dict, dict): # pylint: disable=raise-missing-from raise TaskCatException( - 'Invalid parameter %s parameters must be of type dict ("{}")' - % param_dict + f'Invalid parameter {param_dict}: parameters must be of type dict ("{{}}")' ) + + # Validate required keys are present if "ParameterKey" not in param_dict or "ParameterValue" not in param_dict: # pylint: disable=raise-missing-from raise TaskCatException( - f"Invalid parameter {param_dict} all items must " - f"have both ParameterKey and ParameterValue keys" + f"Invalid parameter {param_dict}: all items must " + f"have both 'ParameterKey' and 'ParameterValue' keys" ) + + # Add parameter to index mapping key = param_dict["ParameterKey"] param_index[key] = idx + return param_index def merge_dicts(list_of_dicts): + """ + Merge multiple dictionaries into a single dictionary. + + Later dictionaries in the list will override values from earlier ones + if there are key conflicts. + + Args: + list_of_dicts (list): List of dictionaries to merge + + Returns: + dict: Merged dictionary containing all key-value pairs + + Example: + >>> dicts = [{'a': 1, 'b': 2}, {'b': 3, 'c': 4}] + >>> merge_dicts(dicts) + {'a': 1, 'b': 3, 'c': 4} + """ merged_dict = {} for single_dict in list_of_dicts: merged_dict = {**merged_dict, **single_dict} @@ -154,89 +402,266 @@ def merge_dicts(list_of_dicts): def pascal_to_snake(pascal): - sub = ALL_CAP_RE.sub(r"\1_\2", pascal) + """ + Convert PascalCase string to snake_case. + + Args: + pascal (str): String in PascalCase format + + Returns: + str: String converted to snake_case + + Example: + >>> pascal_to_snake('MyVariableName') + 'my_variable_name' + >>> pascal_to_snake('HTTPSConnection') + 'https_connection' + """ + # First pass: handle sequences like 'HTTPSConnection' -> 'HTTPS_Connection' + sub = FIRST_CAP_RE.sub(r"\1_\2", pascal) + # Second pass: handle remaining cases and convert to lowercase return ALL_CAP_RE.sub(r"\1_\2", sub).lower() def merge_nested_dict(old, new): + """ + Recursively merge nested dictionaries. + + This function performs a deep merge of two dictionaries, where nested + dictionaries are merged recursively rather than being replaced entirely. + + Args: + old (dict): The base dictionary to merge into (modified in-place) + new (dict): The dictionary to merge from + + Note: + This function modifies the 'old' dictionary in-place. + + Example: + >>> old = {'a': {'x': 1, 'y': 2}, 'b': 3} + >>> new = {'a': {'y': 20, 'z': 30}, 'c': 4} + >>> merge_nested_dict(old, new) + >>> print(old) + {'a': {'x': 1, 'y': 20, 'z': 30}, 'b': 3, 'c': 4} + """ for k, v in new.items(): + # If both values are dictionaries, merge recursively if isinstance(old.get(k), dict) and isinstance(v, collections.abc.Mapping): merge_nested_dict(old[k], v) else: + # Otherwise, replace the value old[k] = v def ordered_dump(data, stream=None, dumper=yaml.Dumper, **kwds): + """ + Dump YAML while preserving the order of OrderedDict objects. + + Standard YAML dumping doesn't preserve the order of OrderedDict objects. + This function creates a custom dumper that maintains the order. + + Args: + data: The data structure to dump to YAML + stream: Output stream (file-like object) or None for string output + dumper: YAML dumper class to extend. Defaults to yaml.Dumper + **kwds: Additional keyword arguments passed to yaml.dump + + Returns: + str or None: YAML string if stream is None, otherwise None + + Example: + >>> from collections import OrderedDict + >>> data = OrderedDict([('first', 1), ('second', 2)]) + >>> yaml_str = ordered_dump(data) + >>> print(yaml_str) + first: 1 + second: 2 + """ class OrderedDumper(dumper): # pylint: disable=too-many-ancestors + """Custom YAML dumper that preserves OrderedDict order.""" pass def _dict_representer(dumper, data): + """Represent OrderedDict as a regular mapping while preserving order.""" return dumper.represent_mapping( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items() ) + # Register the custom representer for OrderedDict OrderedDumper.add_representer(OrderedDict, _dict_representer) return yaml.dump(data, stream, OrderedDumper, **kwds) def deep_get(dictionary, keys, default=None): - zulu = reduce( + """ + Get a value from a nested dictionary using a path-like key string. + + This function allows accessing nested dictionary values using a slash-separated + path string, similar to file system paths. + + Args: + dictionary (dict): The dictionary to search in + keys (str): Slash-separated path to the desired value (e.g., 'level1/level2/key') + default: Default value to return if the path is not found + + Returns: + The value at the specified path, or the default value if not found + + Example: + >>> data = {'config': {'database': {'host': 'localhost', 'port': 5432}}} + >>> deep_get(data, 'config/database/host') + 'localhost' + >>> deep_get(data, 'config/cache/ttl', 300) + 300 + """ + return reduce( lambda d, key: d.get(key, default) if isinstance(d, dict) else default, keys.split("/"), dictionary, ) - return zulu def neglect_submodule_templates(project_root, template_list): + """ + Filter out CloudFormation templates that are located within Git submodules. + + This function examines the project's .gitmodules file to identify submodule + paths and removes any templates that are located within those submodules. + This prevents TaskCat from processing templates that are part of external + repositories included as submodules. + + Args: + project_root (Path): Path to the project root directory + template_list (list): List of template objects to filter + + Returns: + list: Filtered list of template objects excluding those in submodules + + Note: + If no .gitmodules file is found, the original template list is returned unchanged. + """ template_dict = {} - # one template object per path. + + # Create a dictionary mapping template paths to template objects + # Include both main templates and their descendants for template in template_list: template_dict[template.template_path] = template for template_descendent in template.descendents: template_dict[template_descendent.template_path] = template_descendent - # Removing those within a submodule. + # Get submodule path prefixes from .gitmodules submodule_path_prefixes = [] try: gitmodule_config = ConfigFile.from_path(Path(project_root / ".gitmodules")) except FileNotFoundError: + # No .gitmodules file found, return original list return template_list + # Parse submodule paths from the configuration for submodule_path, _, _ in parse_submodules(gitmodule_config): submodule_path_prefixes.append( Path(project_root / submodule_path.decode("utf-8")) ) + # Filter out templates that are within submodule directories finalized_templates = [] for template_obj in list(template_dict.values()): gitmodule_template = False + + # Check if this template is within any submodule path for gm_path in submodule_path_prefixes: if gm_path in template_obj.template_path.parents: gitmodule_template = True + break + + # Only include templates that are not in submodules if not gitmodule_template: finalized_templates.append(template_obj) + return finalized_templates def determine_profile_for_region(auth_dict, region): + """ + Determine the appropriate AWS profile to use for a specific region. + + This function looks up the AWS profile configuration for a given region, + falling back to the default profile if no region-specific profile is configured. + + Args: + auth_dict (dict): Dictionary mapping regions to AWS profile names + region (str): AWS region code to look up + + Returns: + str: AWS profile name to use for the specified region + + Example: + >>> auth_config = { + ... 'us-east-1': 'prod-profile', + ... 'us-west-2': 'dev-profile', + ... 'default': 'default-profile' + ... } + >>> determine_profile_for_region(auth_config, 'us-east-1') + 'prod-profile' + >>> determine_profile_for_region(auth_config, 'eu-west-1') + 'default-profile' + """ profile = auth_dict.get(region, auth_dict.get("default", "default")) return profile def fetch_ssm_parameter_value(boto_client, parameter_path): + """ + Fetch a parameter value from AWS Systems Manager Parameter Store. + + Args: + boto_client: Boto3 client factory function + parameter_path (str): The parameter path/name in SSM Parameter Store + + Returns: + str: The parameter value from SSM + + Raises: + ClientError: If the parameter doesn't exist or access is denied + + Example: + >>> value = fetch_ssm_parameter_value(boto_client, '/myapp/database/password') + >>> print(value) + 'secret-password-value' + """ ssm = boto_client("ssm") response = ssm.get_parameter(Name=parameter_path) return response["Parameter"]["Value"] def fetch_secretsmanager_parameter_value(boto_client, secret_arn): + """ + Fetch a secret value from AWS Secrets Manager. + + Args: + boto_client: Boto3 client factory function + secret_arn (str): The ARN or name of the secret in Secrets Manager + + Returns: + str: The secret value from Secrets Manager + + Raises: + TaskCatException: If the secret cannot be retrieved or doesn't exist + + Example: + >>> secret = fetch_secretsmanager_parameter_value( + ... boto_client, + ... 'arn:aws:secretsmanager:us-east-1:123456789012:secret:MySecret-AbCdEf' + ... ) + >>> print(secret) + '{"username": "admin", "password": "secret123"}' + """ secrets_manager = boto_client("secretsmanager") try: response = secrets_manager.get_secret_value(SecretId=secret_arn)["SecretString"] except Exception as e: # pylint: disable=raise-missing-from raise TaskCatException( - "ARN: {} encountered an error: {}".format(secret_arn, str(e)) + f"Failed to retrieve secret from ARN '{secret_arn}': {str(e)}" ) return response diff --git a/taskcat/_logger.py b/taskcat/_logger.py index dd99d1a33..3b1e8d385 100644 --- a/taskcat/_logger.py +++ b/taskcat/_logger.py @@ -1,46 +1,140 @@ +""" +TaskCat Logging Module + +This module provides custom logging functionality for TaskCat with colored output +and specialized formatting. It includes ANSI color codes for terminal output and +custom log level formatting to provide a consistent and visually appealing +command-line interface. + +The logging system supports different log levels with distinct colors and provides +special formatting for TaskCat-specific operations like S3 interactions. +""" + import logging +# Initialize module-level logger LOG = logging.getLogger(__name__) class PrintMsg: - header = "\x1b[1;41;0m" - highlight = "\x1b[0;30;47m" - name_color = "\x1b[0;37;44m" - aqua = "\x1b[0;30;46m" - green = "\x1b[0;30;42m" - white = "\x1b[0;30;47m" - orange = "\x1b[0;30;43m" - red = "\x1b[0;30;41m" - rst_color = "\x1b[0m" + """ + ANSI color codes and formatted log level strings for terminal output. + + This class provides consistent color coding across the TaskCat CLI interface. + Each log level has its own color scheme to help users quickly identify + different types of messages. + + Color Attributes: + header: Red background for headers + highlight: White background for highlighting + name_color: Blue background for name tags + aqua: Cyan background for debug messages + green: Green background for success messages + white: White background for info messages + orange: Yellow background for warnings + red: Red background for errors + rst_color: Reset color code to return to default + + Formatted Log Levels: + Each log level includes color coding and consistent spacing for alignment + """ + + # ANSI color code definitions + header = "\x1b[1;41;0m" # Bold red background + highlight = "\x1b[0;30;47m" # Black text on white background + name_color = "\x1b[0;37;44m" # White text on blue background + aqua = "\x1b[0;30;46m" # Black text on cyan background + green = "\x1b[0;30;42m" # Black text on green background + white = "\x1b[0;30;47m" # Black text on white background + orange = "\x1b[0;30;43m" # Black text on yellow background + red = "\x1b[0;30;41m" # Black text on red background + rst_color = "\x1b[0m" # Reset to default colors + + # Formatted log level strings with consistent spacing CRITICAL = "{}[FATAL ]{} : ".format(red, rst_color) ERROR = "{}[ERROR ]{} : ".format(red, rst_color) DEBUG = "{}[DEBUG ]{} : ".format(aqua, rst_color) PASS = "{}[PASS ]{} : ".format(green, rst_color) INFO = "{}[INFO ]{} : ".format(white, rst_color) WARNING = "{}[WARN ]{} : ".format(orange, rst_color) + + # Special formatting for TaskCat branding NAMETAG = "{1}{0}{2}".format("taskcat", name_color, rst_color) + + # Special formatting for S3 operations S3 = "{}[S3: -> ]{} ".format(white, rst_color) S3DELETE = "{}[S3: DELETE ]{} ".format(white, rst_color) class AppFilter(logging.Filter): + """ + Custom logging filter to add color formatting to log records. + + This filter processes each log record and adds a 'color_loglevel' attribute + that contains the appropriately colored log level string. It allows for + custom formatting on a per-record basis by checking for special attributes. + """ + def filter(self, record): + """ + Process a log record and add color formatting. + + Args: + record (logging.LogRecord): The log record to process + + Returns: + bool: Always returns True to allow the record to be processed + """ + # Check if this record has a custom nametag (for special formatting) if "nametag" in dir(record): record.color_loglevel = record.nametag else: + # Use the standard colored log level based on the record's level record.color_loglevel = getattr(PrintMsg, record.levelname) + return True def init_taskcat_cli_logger(loglevel=None): + """ + Initialize and configure the TaskCat CLI logger with colored output. + + Sets up a logger with custom formatting that includes ANSI color codes + for terminal output. The logger uses a custom filter to add color + information to each log record. + + Args: + loglevel (str, optional): The logging level to set. Can be any standard + logging level name (DEBUG, INFO, WARNING, ERROR, CRITICAL). + If not provided, the logger level is not explicitly set. + + Returns: + logging.Logger: Configured logger instance ready for use + + Example: + >>> logger = init_taskcat_cli_logger('INFO') + >>> logger.info("This will appear with colored formatting") + """ + # Get the logger for the TaskCat package log = logging.getLogger(__package__) + + # Create a stream handler for console output cli_handler = logging.StreamHandler() + + # Set up custom formatter that uses the color_loglevel attribute formatter = logging.Formatter("%(color_loglevel)s%(message)s") cli_handler.setFormatter(formatter) + + # Add the custom filter to inject color information cli_handler.addFilter(AppFilter()) + + # Attach the handler to the logger log.addHandler(cli_handler) + + # Set the log level if provided if loglevel: - loglevel = getattr(logging, loglevel.upper(), 20) + # Convert string level name to numeric level + loglevel = getattr(logging, loglevel.upper(), 20) # Default to INFO (20) if invalid log.setLevel(loglevel) + return log diff --git a/taskcat/_name_generator.py b/taskcat/_name_generator.py index f6d5afc8d..74ec80211 100644 --- a/taskcat/_name_generator.py +++ b/taskcat/_name_generator.py @@ -1,3 +1,16 @@ +""" +TaskCat Name Generator Module + +This module provides functionality to generate random, human-readable names for +TaskCat resources such as CloudFormation stacks, S3 buckets, and other AWS resources. + +The names are generated by combining random descriptive adjectives with animal names, +creating memorable and unique identifiers like "clever-dolphin" or "swift-eagle". + +The word lists are stored in external text files to allow for easy customization +and expansion of the available vocabulary. +""" + from pathlib import Path from random import choice @@ -5,11 +18,51 @@ def generate_name(): + """ + Generate a random human-readable name for TaskCat resources. + + Creates a unique name by combining a random descriptor (adjective) with a + random animal name, separated by a hyphen. This provides memorable names + for AWS resources that are easier to identify than random strings. + + The function reads from two dictionary files: + - descriptors.txt: Contains adjectives and descriptive words + - animals.txt: Contains animal names + + Returns: + str: A randomly generated name in the format "descriptor-animal" + (e.g., "clever-dolphin", "swift-eagle") + + Raises: + TaskCatException: If the required dictionary files cannot be found + or accessed + + Example: + >>> name = generate_name() + >>> print(name) + "brave-tiger" + """ + # Resolve the path to the configuration directory containing dictionary files path: Path = (Path(__file__).parent / "./cfg/").resolve() - if not (path / "animals.txt").is_file() or not (path / "descriptors.txt").is_file(): - raise TaskCatException("cannot find dictionary files") - with open(str(path / "animals.txt"), "r", encoding="utf-8") as _f: + + # Verify that both required dictionary files exist + animals_file = path / "animals.txt" + descriptors_file = path / "descriptors.txt" + + if not animals_file.is_file() or not descriptors_file.is_file(): + raise TaskCatException( + f"Cannot find dictionary files. Expected files: " + f"{animals_file} and {descriptors_file}" + ) + + # Read and parse the animals dictionary + with open(str(animals_file), "r", encoding="utf-8") as _f: animals = _f.read().split("\n") - with open(str(path / "descriptors.txt"), "r", encoding="utf-8") as _f: + + # Read and parse the descriptors dictionary + with open(str(descriptors_file), "r", encoding="utf-8") as _f: descriptors = _f.read().split("\n") + + # Generate and return a random combination + # Note: Using random.choice is acceptable here as this is not for cryptographic purposes return choice(descriptors) + "-" + choice(animals) # nosec: B311 diff --git a/taskcat/_template_params.py b/taskcat/_template_params.py index c399d449a..d4eceaa63 100644 --- a/taskcat/_template_params.py +++ b/taskcat/_template_params.py @@ -172,6 +172,8 @@ def transform_parameter(self): # $[taskcat_ssm_X] self._get_ssm_param_value_wrapper(self.RE_SSM_PARAMETER) + # $[taskcat_secretsmanager_X] + self._get_secretsmanager_param_value_wrapper(self.RE_SECRETSMANAGER_PARAMETER) # $[taskcat_current_region] self._regex_replace_param_value( self.RE_CURRENT_REGION, self._gen_current_region() diff --git a/taskcat/exceptions.py b/taskcat/exceptions.py index 847bc2f68..e4638bb02 100644 --- a/taskcat/exceptions.py +++ b/taskcat/exceptions.py @@ -1,14 +1,66 @@ +""" +TaskCat Exception Classes + +This module defines custom exception classes used throughout the TaskCat application. +These exceptions provide specific error handling for various failure scenarios that +can occur during CloudFormation template testing and deployment operations. + +The exception hierarchy allows for granular error handling and provides meaningful +error messages to help users diagnose and resolve issues. +""" + + class TaskCatException(Exception): - """Raised when taskcat experiences a fatal error""" + """ + Base exception class for all TaskCat-specific errors. + + This is the parent class for all custom exceptions in TaskCat. It should be + raised when TaskCat experiences a fatal error that prevents normal operation. + + This exception is typically caught at the CLI level to provide user-friendly + error messages and appropriate exit codes. + + Attributes: + message (str): Human-readable error message describing the issue + """ + + def __init__(self, message="TaskCat encountered a fatal error"): + """ + Initialize the TaskCat exception. + + Args: + message (str): Descriptive error message. Defaults to generic message. + """ + self.message = message + super().__init__(self.message) class InvalidActionError(TaskCatException): - """Exception raised for error when invalid action is supplied - + """ + Exception raised when an invalid action or command is supplied to TaskCat. + + This exception is typically raised during command-line argument parsing or + when validating user input that specifies what action TaskCat should perform. + Attributes: - expression -- input expression in which the error occurred + expression (str): The invalid input expression that caused the error + message (str): Human-readable error message """ - def __init__(self, expression): + def __init__(self, expression, message=None): + """ + Initialize the InvalidActionError exception. + + Args: + expression (str): The invalid input expression that triggered the error + message (str, optional): Custom error message. If not provided, a default + message will be generated using the expression. + """ self.expression = expression - super().__init__() + + # Generate default message if none provided + if message is None: + message = f"Invalid action or expression: '{expression}'" + + self.message = message + super().__init__(self.message) diff --git a/taskcat/project_templates/quickstart/README.md.jinja b/taskcat/project_templates/quickstart/README.md.jinja deleted file mode 100644 index ab9fea9ae..000000000 --- a/taskcat/project_templates/quickstart/README.md.jinja +++ /dev/null @@ -1,3 +0,0 @@ -# {{ config.project_name.upper() }} - -This is the readme for {{ config.project_name }} diff --git a/travis-specific-requirements.txt b/travis-specific-requirements.txt deleted file mode 100644 index 622b7abcd..000000000 --- a/travis-specific-requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -reprint -tabulate>=0.8.2,<1.0 -cfn_lint>=0.78.1,<2.0 -setuptools>=40.4.3 -boto3>=1.9.21,<2.0 -botocore>=1.12.21,<2.0 -yattag>=1.10.0,<2.0 -PyYAML~=5.1 -jinja2>=3.1.1,<4.0 -requests>2.17.0 -jsonschema~=3.0 -docker~=4.0 -dulwich~=0.19 -dataclasses;python_version<"3.7" -dataclasses-jsonschema>=2.9.0,<3.0 -pip diff --git a/update_local_zones.py b/update_local_zones.py index 3146ab8b0..8099c864e 100755 --- a/update_local_zones.py +++ b/update_local_zones.py @@ -1,19 +1,87 @@ +#!/usr/bin/env python +""" +AWS Local Zones Updater + +This script automatically updates the list of AWS Availability Zones and Local Zones +used by TaskCat. It queries all AWS regions to discover available zones and generates +a Python module with the current zone mappings. + +AWS Zone Types: +- availability-zone: Standard AZs within AWS regions +- local-zone: AWS Local Zones for ultra-low latency applications + +The script uses zone IDs (e.g., 'use1-az1') rather than zone names (e.g., 'us-east-1a') +because zone names can vary between AWS accounts while zone IDs are consistent. + +Usage: + python update_local_zones.py + +Output: + Updates ./taskcat/local_zones.py with current zone mappings + +Requirements: + - AWS credentials configured (via AWS CLI, environment variables, or IAM role) + - EC2 describe permissions for all regions +""" + import json import boto3 +# Initialize EC2 client for the default region to get list of all regions +print("Initializing AWS EC2 client...") ec2 = boto3.client("ec2") +# Set to store unique zone IDs (using set to avoid duplicates) local_zones = set() + +# Get list of all AWS regions +print("Fetching list of AWS regions...") regions = [x["RegionName"] for x in ec2.describe_regions()["Regions"]] +print(f"Found {len(regions)} regions to query") -for rn in regions: - e = boto3.client("ec2", region_name=rn) - for zone in e.describe_availability_zones(AllAvailabilityZones=True)[ - "AvailabilityZones" - ]: - if zone["ZoneType"] in ["availability-zone", "local-zone"]: - local_zones.add(zone["ZoneId"]) +# Query each region for its availability zones and local zones +for region_name in regions: + print(f"Querying zones in region: {region_name}") + + try: + # Create region-specific EC2 client + regional_ec2 = boto3.client("ec2", region_name=region_name) + + # Get all availability zones (including local zones) for this region + # AllAvailabilityZones=True includes local zones and wavelength zones + zones_response = regional_ec2.describe_availability_zones(AllAvailabilityZones=True) + + # Process each zone in the region + for zone in zones_response["AvailabilityZones"]: + # Only include standard availability zones and local zones + # Exclude wavelength zones and other specialized zone types + if zone["ZoneType"] in ["availability-zone", "local-zone"]: + # Use ZoneId (e.g., 'use1-az1') instead of ZoneName (e.g., 'us-east-1a') + # ZoneId is consistent across AWS accounts while ZoneName can vary + local_zones.add(zone["ZoneId"]) + + except Exception as e: + print(f"Warning: Failed to query region {region_name}: {e}") + # Continue with other regions even if one fails + continue +print(f"Discovered {len(local_zones)} unique zones across all regions") + +# Write the zone mappings to the Python module file +print("Writing zone mappings to local_zones.py...") with open("./taskcat/local_zones.py", "w") as fh: + # Write file header comment + fh.write('"""\n') + fh.write("AWS Availability Zones and Local Zones Mappings\n\n") + fh.write("This file is automatically generated by update_local_zones.py\n") + fh.write("Do not edit manually - run the update script instead.\n\n") + fh.write("Contains zone IDs (e.g., 'use1-az1') for all availability zones\n") + fh.write("and local zones across all AWS regions.\n") + fh.write('"""\n\n') + + # Write the zones list, sorted for consistency + fh.write("# List of all AWS availability zone and local zone IDs\n") fh.write(f"ZONES = {json.dumps(sorted(local_zones), indent=4)}\n\n") + +print("Successfully updated local zones mappings!") diff --git a/update_partition_region_map.py b/update_partition_region_map.py index 7b8521d5c..660cc5a13 100755 --- a/update_partition_region_map.py +++ b/update_partition_region_map.py @@ -1,26 +1,84 @@ #!/usr/bin/env python -# because something like https://github.com/boto/boto3/issues/1868 doesn't exist +""" +AWS Region to Partition Mapping Updater + +This script automatically updates the AWS region-to-partition mapping used by TaskCat. +It fetches the latest endpoint data from the official boto/botocore repository and +generates a Python module with current region and partition mappings. + +The script addresses the lack of a programmatic way to get region-partition mappings +from boto3/botocore (see: https://github.com/boto/boto3/issues/1868). + +AWS Partitions: +- aws: Standard AWS regions (most common) +- aws-cn: AWS China regions +- aws-us-gov: AWS GovCloud regions + +Usage: + python update_partition_region_map.py + +Output: + Updates ./taskcat/regions_to_partitions.py with current mappings + +The generated file contains: +- REGIONS: Dict mapping region codes to partition names +- PARTITIONS: Dict mapping partition names to lists of regions +""" import json import requests +# URL to the official AWS endpoints data from botocore ENDPOINT_JSON = ( "https://raw.githubusercontent.com/boto/botocore/master/botocore/" "data/endpoints.json" ) if __name__ == "__main__": + # Fetch the latest endpoints data from botocore repository + print("Fetching latest AWS endpoints data...") resp = requests.get(ENDPOINT_JSON) + + # Check if the request was successful if resp.status_code != 200: - raise Exception(f"{resp.status_code} {resp.reason}") + raise Exception(f"Failed to fetch endpoints data: {resp.status_code} {resp.reason}") + + # Parse the JSON response endpoints = resp.json() - partitions = {} - regions = {} - for p in endpoints["partitions"]: - partitions[p["partition"]] = list(p["regions"]) - for r in p["regions"]: - regions[r] = p["partition"] + + # Initialize dictionaries for partition and region mappings + partitions = {} # partition_name -> [list of regions] + regions = {} # region_name -> partition_name + + # Process each partition in the endpoints data + for partition_data in endpoints["partitions"]: + partition_name = partition_data["partition"] + partition_regions = list(partition_data["regions"]) + + # Store partition -> regions mapping + partitions[partition_name] = partition_regions + + # Store region -> partition mapping for each region + for region in partition_data["regions"]: + regions[region] = partition_name + + # Write the mappings to the Python module file + print(f"Writing mappings for {len(regions)} regions across {len(partitions)} partitions...") with open("./taskcat/regions_to_partitions.py", "w") as fh: - fh.write(f"REGIONS = {json.dumps(regions, indent=4)}\n\n") - fh.write(f"PARTITIONS = {json.dumps(partitions, indent=4)}\n") + # Write file header comment + fh.write('"""\n') + fh.write("AWS Region to Partition Mappings\n\n") + fh.write("This file is automatically generated by update_partition_region_map.py\n") + fh.write("Do not edit manually - run the update script instead.\n") + fh.write('"""\n\n') + + # Write the regions dictionary (region -> partition mapping) + fh.write("# Mapping of AWS region codes to their partition names\n") + fh.write(f"REGIONS = {json.dumps(regions, indent=4, sort_keys=True)}\n\n") + + # Write the partitions dictionary (partition -> regions mapping) + fh.write("# Mapping of AWS partition names to their region lists\n") + fh.write(f"PARTITIONS = {json.dumps(partitions, indent=4, sort_keys=True)}\n") + + print("Successfully updated region-to-partition mappings!")