Fix Transformers + Cuda Context bug in NeMo Curator #2329

	name: Test Python package
	on:
	push:
	branches:
	- main
	pull_request:
	workflow_dispatch:

	# When this workflow is queued, automatically cancel any previous running
	# or pending jobs from the same branch
	concurrency:
	group: test-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	build_and_test:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ["3.10", "3.12"]
	steps:
	- uses: actions/checkout@v4
	- name: Optionally free up space on Ubuntu
	run: \|
	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /opt/ghc
	sudo rm -rf "/usr/local/share/boost"
	sudo rm -rf "$AGENT_TOOLSDIRECTORY"
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install NeMo-Curator and pytest
	# TODO: Remove pytest when optional test dependencies are added to setup.py

	# Installing wheel beforehand due to fasttext issue:
	# https://github.com/facebookresearch/fastText/issues/512#issuecomment-1837367666
	run: \|
	pip install -U pip
	pip install --no-cache-dir wheel
	pip install --no-cache-dir .
	pip install --no-cache-dir pytest coverage

	- name: Run tests
	run: \|
	python -m coverage run --branch --source=nemo_curator --omit=nemo_curator/scripts/* -m pytest -v --cpu

	- name: Generate report
	run: \|
	python -m coverage xml
	python -m coverage report

	- name: Upload coverage reports to Codecov
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	files: coverage.xml
	verbose: true
	flags: cpu

Provide feedback