test: improve testcafe test flakiness, use config file (#1627)

.testcaferc.json

@@ -0,0 +1,9 @@
+{
+  "hostname": "localhost",
+  "quarantineMode": true,
+  "screenshots": {
+    "takeOnFails": false
+  },
+  "skipJsErrors": true,
+  "stopOnFirstFail": true
+}

CONTRIBUTING.md

@@ -77,7 +77,7 @@ In separate terminals:
 
 3\. Run a debuggable TestCafé instance:
 
-    npx testcafe --hostname localhost --skip-js-errors --debug-mode chrome tests/spec
+    npx testcafe --debug-mode chrome tests/spec
 
 ### Test conventions
 

package.json

@@ -26,8 +26,8 @@
     "test-browser-suite1": "run-p --race run-mastodon build-and-start test-mastodon-suite1",
     "test-mastodon-suite1": "run-s wait-for-mastodon-to-start wait-for-mastodon-data testcafe-suite1",
     "testcafe": "run-s testcafe-suite0 testcafe-suite1",
-    "testcafe-suite0": "cross-env-shell testcafe --hostname localhost --skip-js-errors -q -c 4 $BROWSER tests/spec/0*",
-    "testcafe-suite1": "cross-env-shell testcafe --hostname localhost --skip-js-errors -q $BROWSER tests/spec/1*",
+    "testcafe-suite0": "cross-env-shell testcafe -c 4 $BROWSER tests/spec/0*",
+    "testcafe-suite1": "cross-env-shell testcafe $BROWSER tests/spec/1*",
     "test-unit": "mocha -r esm -r bin/browser-shim.js tests/unit/",
     "wait-for-mastodon-to-start": "node -r esm bin/wait-for-mastodon-to-start.js",
     "wait-for-mastodon-data": "node -r esm bin/wait-for-mastodon-data.js",
@@ -119,7 +119,7 @@
     "mocha": "^6.2.1",
     "now": "^16.3.1",
     "standard": "^14.3.1",
-    "testcafe": "^1.5.0"
+    "testcafe": "^1.6.1"
   },
   "engines": {
     "node": ">= 8"

tests/spec/100-favorite-unfavorite.js

@@ -1,83 +1,97 @@
 import {
   getFavoritesCount,
-  getNthFavoriteButton, getNthFavorited, getNthStatus, getUrl, homeNavButton, notificationsNavButton,
-  scrollToBottom, scrollToTop, sleep
+  getNthFavoriteButton,
+  getNthFavorited,
+  getNthStatus,
+  getNthStatusContent,
+  getUrl,
+  homeNavButton,
+  notificationsNavButton,
+  scrollToBottom,
+  scrollToTop,
+  sleep
 } from '../utils'
 import { loginAsFoobar } from '../roles'
-
-import { homeTimeline } from '../fixtures'
+import { favoriteStatusAs, postAs } from '../serverActions'
 
 fixture`100-favorite-unfavorite.js`
   .page`http://localhost:4002`
 
 test('favorites a status', async t => {
+  await postAs('admin', 'favorite me!!!')
   await loginAsFoobar(t)
   await t
-    .hover(getNthStatus(5))
-    .expect(getNthFavorited(5)).eql('false')
-    .click(getNthFavoriteButton(5))
-    .expect(getNthFavorited(5)).eql('true')
+    .expect(getNthStatusContent(1).innerText).contains('favorite me')
+    .hover(getNthStatus(1))
+    .expect(getNthFavorited(1)).eql('false')
+    .click(getNthFavoriteButton(1))
+    .expect(getNthFavorited(1)).eql('true')
 
   // scroll down and back up to force an unrender
   await scrollToBottom()
   await sleep(1)
   await scrollToTop()
   await t
-    .hover(getNthStatus(5))
-    .expect(getNthFavorited(5)).eql('true')
+    .hover(getNthStatus(1))
+    .expect(getNthFavorited(1)).eql('true')
     .click(notificationsNavButton)
     .click(homeNavButton)
-    .expect(getNthFavorited(5)).eql('true')
+    .expect(getNthFavorited(1)).eql('true')
     .click(notificationsNavButton)
     .expect(getUrl()).contains('/notifications')
     .click(homeNavButton)
     .expect(getUrl()).eql('http://localhost:4002/')
-    .hover(getNthStatus(5))
-    .expect(getNthFavorited(5)).eql('true')
-    .click(getNthFavoriteButton(5))
-    .expect(getNthFavorited(5)).eql('false')
+    .hover(getNthStatus(1))
+    .expect(getNthFavorited(1)).eql('true')
+    .click(getNthFavoriteButton(1))
+    .expect(getNthFavorited(1)).eql('false')
 })
 
 test('unfavorites a status', async t => {
+  const { id: statusId } = await postAs('admin', 'favorite this one too')
+  await favoriteStatusAs('foobar', statusId)
   await loginAsFoobar(t)
   await t
-    .expect(getNthFavorited(2)).eql('true')
-    .click(getNthFavoriteButton(2))
-    .expect(getNthFavorited(2)).eql('false')
+    .expect(getNthStatusContent(1).innerText).contains('favorite this one too')
+    .expect(getNthFavorited(1)).eql('true')
+    .click(getNthFavoriteButton(1))
+    .expect(getNthFavorited(1)).eql('false')
 
   // scroll down and back up to force an unrender
   await scrollToBottom()
   await sleep(1)
   await scrollToTop()
   await t
-    .expect(getNthFavorited(2)).eql('false')
+    .expect(getNthFavorited(1)).eql('false')
     .click(notificationsNavButton)
     .click(homeNavButton)
-    .expect(getNthFavorited(2)).eql('false')
+    .expect(getNthFavorited(1)).eql('false')
     .click(notificationsNavButton)
     .navigateTo('/')
-    .expect(getNthFavorited(2)).eql('false')
-    .click(getNthFavoriteButton(2))
-    .expect(getNthFavorited(2)).eql('true')
+    .expect(getNthFavorited(1)).eql('false')
+    .click(getNthFavoriteButton(1))
+    .expect(getNthFavorited(1)).eql('true')
 })
 
 test('Keeps the correct favorites count', async t => {
+  const { id: statusId } = await postAs('admin', 'favorite this twice pls')
+  await favoriteStatusAs('quux', statusId)
   await loginAsFoobar(t)
-  const idx = homeTimeline.findIndex(_ => _.content === 'this is unlisted')
   await t
-    .hover(getNthStatus(1 + idx))
-    .click(getNthFavoriteButton(1 + idx))
-    .expect(getNthFavorited(1 + idx)).eql('true')
-    .click(getNthStatus(1 + idx))
+    .expect(getNthStatusContent(1).innerText).contains('favorite this twice pls')
+    .hover(getNthStatus(1))
+    .click(getNthFavoriteButton(1))
+    .expect(getNthFavorited(1)).eql('true')
+    .click(getNthStatus(1))
     .expect(getUrl()).contains('/status')
     .expect(getNthFavorited(1)).eql('true')
     .expect(getFavoritesCount()).eql(2)
     .click(homeNavButton)
     .expect(getUrl()).eql('http://localhost:4002/')
-    .hover(getNthStatus(1 + idx))
-    .click(getNthFavoriteButton(1 + idx))
-    .expect(getNthFavorited(1 + idx)).eql('false')
-    .click(getNthStatus(1 + idx))
+    .hover(getNthStatus(1))
+    .click(getNthFavoriteButton(1))
+    .expect(getNthFavorited(1)).eql('false')
+    .click(getNthStatus(1))
     .expect(getUrl()).contains('/status')
     .expect(getNthFavorited(1)).eql('false')
     .expect(getFavoritesCount()).eql(1)