Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(automation): run integration tests and report drift
Browse files Browse the repository at this point in the history
sanderblue committed May 23, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 3e42961 commit 65bd462
Showing 5 changed files with 140 additions and 150 deletions.
99 changes: 33 additions & 66 deletions .github/workflows/test_integration.yml
Original file line number Diff line number Diff line change
@@ -5,9 +5,6 @@ on:
push:
branches: [main]
pull_request:
# schedule:
# # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm)
# - cron: "0 8,12,16 * * 1-5"

jobs:
test-integration:
@@ -18,68 +15,38 @@ jobs:
with:
go-version: 1.21.x

# - name: Add GOBIN to PATH
# run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
# shell: bash
- name: Add GOBIN to PATH
run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
shell: bash

# - name: Checkout code
# uses: actions/checkout@v4

# - name: Cache deps
# uses: actions/cache@v4
# with:
# path: ~/go/pkg/mod
# key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
# restore-keys: |
# ${{ runner.os }}-go-

# - name: Integration Tests
# if: github.event.pull_request.head.repo.full_name == github.repository
# run: make test-integration cover-report
# env:
# NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
# NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }}
# NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }}
# NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }}
# NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
# NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }}
# NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }}
# NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }}
# INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }}
# INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }}
# INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }}
# INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}}
# INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }}
# INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }}
# INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }}
# INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }}

# - name: Create test failure report
# id: testFailureReport
# run: |
# go install github.com/mfridman/tparse@latest
# make test-failure-report
# tparse -file=coverage/integration.failures -format=markdown >> "$GITHUB_OUTPUT"

# - name: Send test failures report to Slack
# id: slack
# uses: slackapi/[email protected]
# with:
# # Uses Slack's Block Kit to build the message
# payload: |
# {
# "text": "New Relic Terraform Provider Test Failures",
# "blocks": [
# {
# "type": "section",
# "text": {
# "type": "mrkdwn",
# "text": "${{ steps.testFailureReport.outputs }}"
# }
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
# SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
- name: Checkout code
uses: actions/checkout@v4

- name: Cache deps
uses: actions/cache@v4
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Integration Tests
if: github.event.pull_request.head.repo.full_name == github.repository
run: make test-integration cover-report
env:
NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }}
NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }}
NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }}
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }}
NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }}
NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }}
INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }}
INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }}
INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }}
INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}}
INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }}
INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }}
INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }}
INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }}
104 changes: 104 additions & 0 deletions .github/workflows/test_integration_cron.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
name: Integration Tests Cron Job

on:
workflow_dispatch: # this is just to test the functionality, remove before merging!!!!!!!!!!!!
push:
branches: [main]
pull_request:
# schedule:
# # Cron executes at 0800, 1200 1600 (8am, 12pm, 4pm)
# - cron: "0 8,12,16 * * 1-5"

jobs:
test-integration:
# if: github.ref == 'refs/heads/main'
uses: newrelic/terraform-provider-newrelic/.github/workflows/test_integration.yml@main
secrets:
NEW_RELIC_ACCOUNT_ID: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
NEW_RELIC_SUBACCOUNT_ID: ${{ secrets.NEW_RELIC_SUBACCOUNT_ID }}
NEW_RELIC_ADMIN_API_KEY: ${{ secrets.NEW_RELIC_ADMIN_API_KEY }}
NEW_RELIC_API_KEY: ${{ secrets.NEW_RELIC_API_KEY }}
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
NEW_RELIC_REGION: ${{ secrets.NEW_RELIC_REGION }}
NEW_RELIC_INSIGHTS_INSERT_KEY: ${{ secrets.NEW_RELIC_INSIGHTS_INSERT_KEY }}
NR_ACC_TESTING: ${{ secrets.NR_ACC_TESTING }}
INTEGRATION_TESTING_AWS_ARN: ${{ secrets.INTEGRATION_TESTING_AWS_ARN }}
INTEGRATION_TESTING_GCP_ACCOUNT_NAME: ${{ secrets.INTEGRATION_TESTING_GCP_ACCOUNT_NAME }}
INTEGRATION_TESTING_GCP_PROJECT_ID: ${{ secrets.INTEGRATION_TESTING_GCP_PROJECT_ID }}
INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID: ${{secrets.INTEGRATION_TESTING_GCP_INTEGRATIONS_PROJECT_ID}}
INTEGRATION_TESTING_AZURE_APPLICATION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_APPLICATION_ID }}
INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_CLIENT_SECRET_ID }}
INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_SUBSCRIPTION_ID }}
INTEGRATION_TESTING_AZURE_TENANT_ID: ${{ secrets.INTEGRATION_TESTING_AZURE_TENANT_ID }}

detect-drift:
runs-on: ubuntu-latest
steps:
- name: Check check
run: ls

- name: Detect drift
if: ${{ always() }}
uses: actions/github-script@v7
id: detect-state-drift
with:
script: |
const script = require('./scripts/detect-state-drift.js')
await script({core})
- name: Send report to Slack
id: slack
if: ${{ steps.detect-state-drift.outputs.drift_report != '' }}
uses: slackapi/[email protected]
with:
# Uses Slack's Block Kit to build the message
# https://app.slack.com/block-kit-builder
payload: |
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":warning: *Terraform Provider | Drift Detected*"
}
},
{
"type": "divider"
},
{
"type": "context",
"elements": [
{
"type": "plain_text",
"text": ${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }}
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "More information can be viewed in the job summary."
},
"accessory": {
"type": "button",
"text": {
"type": "plain_text",
"text": "Job Summary",
"emoji": true
},
"value": "View Job Summary",
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
"action_id": "button-action"
}
},
{
"type": "divider"
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

65 changes: 1 addition & 64 deletions .github/workflows/test_unit.yml
Original file line number Diff line number Diff line change
@@ -35,73 +35,10 @@ jobs:
run: |
make test-unit cover-report
- name: Detect drift
if: ${{ always() }}
uses: actions/github-script@v7
id: detect-state-drift
with:
script: |
const script = require('./scripts/detect-state-drift.js')
await script({core})
# - name: Report unit test coverage via Codecov
# uses: codecov/codecov-action@v3
# with:
# files: ./coverage/coverage.out

- name: Send report to Slack
id: slack
if: ${{ always() }}
uses: slackapi/[email protected]
with:
# Uses Slack's Block Kit to build the message
# https://app.slack.com/block-kit-builder
payload: |
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":warning: *Terraform Provider | Drift Detected*"
}
},
{
"type": "divider"
},
{
"type": "context",
"elements": [
{
"type": "plain_text",
"text": ${{ toJSON(steps.detect-state-drift.outputs.failed_tests_with_drift) }}
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "More information can be viewed in the job summary."
},
"accessory": {
"type": "button",
"text": {
"type": "plain_text",
"text": "Job Summary",
"emoji": true
},
"value": "View Job Summary",
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
"action_id": "button-action"
}
},
{
"type": "divider"
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK


5 changes: 0 additions & 5 deletions build/test.mk
Original file line number Diff line number Diff line change
@@ -35,11 +35,6 @@ test-integration: tools
-- -v -parallel 10 -tags=integration $(TEST_ARGS) -covermode=$(COVERMODE) -coverprofile $(COVERAGE_DIR)/integration.tmp \
-timeout 120m -ldflags=$(LDFLAGS_TEST)

test-failure-report: tools
@echo "=== $(PROJECT_NAME) === [ test-report-failed ]: generating report..."
@mkdir -p $(COVERAGE_DIR)
./scripts/test-failure-report.sh

#
# Coverage
#
17 changes: 2 additions & 15 deletions scripts/detect-state-drift.js
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ module.exports = async ({
const fs = require('fs');

// Read the text from the file
fs.readFile('coverage/unit.report', 'utf8', (err, data) => {
fs.readFile('coverage/integration.report', 'utf8', (err, data) => {
if (err) {
console.error('Error reading file:', err);
return;
@@ -23,7 +23,6 @@ module.exports = async ({
}
});

// Print the resulting JSON array
// console.log(jsonData);

const report = jsonData.filter(data => {
@@ -37,18 +36,6 @@ module.exports = async ({
msg = `'${report.join('\n')}'`;
}

core.setOutput('failed_tests_with_drift', msg);

// // Convert JSON data to string
// const jsonString = JSON.stringify(failedTests, null, 2);

// // Write the JSON data to a file
// fs.writeFile('output.json', jsonString, 'utf8', err => {
// if (err) {
// console.error('Error writing file:', err);
// return;
// }
// console.log('JSON data has been written to output.json');
// });
core.setOutput('drift_report', msg);
});
};

0 comments on commit 65bd462

Please sign in to comment.