Vaishaal/tokenize shuffle generator (#85)

* tokenize shuffle fixes

* tokenize shuffle fixes

* tokenize shuffle fixes specify tmp dir

* tokenize shuffle bug fixes, remove counter from inner loop

* make batch_format=pandas remove extra loop

* added some rudimentary tests to tokenize shuffle

* added tests that actually pass

* change ray version

* grab dummy credentials for aws stuff to work

* change ci version

* ci version update

* wget dummy creds

* wget ci fixes

* add not s3 to make test

* fix pytest -m command

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install
         run: |
           sudo apt-get update
@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ["3.10"]
 
     steps:
     - uses: actions/checkout@v2
@@ -42,6 +42,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install
       run: |
+        mkdir $HOME/.aws/
+        wget https://gist.githubusercontent.com/Vaishaal/f109bfab6a194a93040ae2a19b6be251/raw/7d8026ae234d77ba1ca29b1f9d114c6780308ae4/dummy_creds -O $HOME/.aws/credentials
         sudo apt-get update
         python3 -m venv .env
         source .env/bin/activate

Makefile

@@ -13,7 +13,7 @@ black: ## [Local development] Auto-format python code using black
 
 TEST_ARGS = tests ## set default to run all tests
 test: ## [Local development] Run unit tests
-	python -m pytest -x -s -v $(TEST_ARGS) -m "not gpu"
+	python -m pytest -x -s -v $(TEST_ARGS) -m "not gpu and not s3"
 
 test-gpu: ## [Local development] Run unit tests
 	python -m pytest -x -s -v $(TEST_ARGS) -m gpu