Scraping Pipeline #7743

	name: Scraping Pipeline

	on:
	schedule:
	# Run every 5 minutes
	- cron: "/6 * * *"
	workflow_dispatch: # Allows manual triggering of the workflow from the Actions tab

	permissions:
	contents: write # Grants permission to push commits

	jobs:
	run-scripts:
	runs-on: ubuntu-latest

	steps:
	- name: Checkout the repository
	uses: actions/checkout@v3

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.10'

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt

	- name: Scraping Data
	run: python several_requests.py

	- name: Filtering Data to your preferences
	run: python filter_datatset.py

	- name: Updating ReadMe
	env:
	BOT_API_KEY: ${{ secrets.BOT_API_KEY }}
	CHANNEL_ID: ${{ secrets.CHANNEL_ID }}
	run: python update_readme.py

	- name: Commit and push changes
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use the GitHub token for authentication
	run: \|
	git config --global user.name "github-actions[bot]"
	git config --global user.email "github-actions[bot]@users.noreply.github.com"
	git add .
	commit_output=$(git commit -m "Auto-commit: Generated files after running Python scripts" \|\| true)
	echo "$commit_output"
	if [[ "$commit_output" != "nothing to commit, working tree clean" ]]; then
	git push
	else
	echo "No changes to push"
	fi

Provide feedback