Visualize robot motion

README.md

@@ -9,3 +9,5 @@ This repository contains code for the feeding web app. The app itself is in `fee
 - [web_video_server (branch: `ros2`)](https://github.com/RobotWebTools/web_video_server/tree/ros2)
     - Dependency: [async_web_server_cpp (branch: `ros2-develop`)](https://github.com/fkie/async_web_server_cpp)
     - Dependency: [vision_opencv (branch: `humble` or your ROS2 version)](https://github.com/ros-perception/vision_opencv/tree/humble)
+
+All these repositories of `feeding_web_interface`, `ada_feeding`, `async_web_server_cpp`, `vision_opencv`, `web_video_server`, and `PRL fork of rosbridge_suite` as mentioned above should be downloaded using `git clone ...` in the "src" folder inside the ROS2 workspace. Please follow the [Ubuntu (Debian) tutorial](https://docs.ros.org/en/humble/Installation/Ubuntu-Install-Debians.html) for ROS2 installation in Ubuntu 22, which will automatically lead to creation of "src" folder in ROS2 workspace. To access hidden `.env` file in Ubuntu to change the debug flag's value, press Ctrl + H in the `feeding_web_interface` folder. 

feedingwebapp/.eslintignore

@@ -4,6 +4,8 @@
 /node_modules
 /.pnp
 .pnp.js
+# don't lint progressbar.js because it is a third-party
+/src/Pages/Home/MealStates/progressbar.js
 
 # testing
 /coverage

feedingwebapp/Acknowledgements.md

@@ -1,3 +1,3 @@
 **Acknowledgement for all the icons we used:**
 
-“[Bowl with fruits](https://thenounproject.com/icon/bowl-with-fruits-5039233/)” icon by Langtik, “[Fork](https://thenounproject.com/icon/fork-10962/)” icon by Dmitry Baranovskiy, “[Human head mouth](https://thenounproject.com/icon/open-mouth-2885042/)” icon by Wuppdidu, “[Human head](https://thenounproject.com/icon/head-2288243/)” icon by Wildson Rafalski, “[Strawberry](https://thenounproject.com/icon/strawberry-3487721/)”icon by Ashabul Kahfi, “[Robot arm](https://thenounproject.com/icon/robot-arm-4466147/)” icon by Peter Lakenbrink, and “[Stop](https://thenounproject.com/icon/stop-34715/)” icon by AS Design from Noun Project CC BY 3.0.
+“[Bowl with fruits](https://thenounproject.com/icon/bowl-with-fruits-5039233/)” icon by Langtik, “[Fork](https://thenounproject.com/icon/fork-10962/)” icon by Dmitry Baranovskiy, “[Human head mouth](https://thenounproject.com/icon/open-mouth-2885042/)” icon by Wuppdidu, “[Human head](https://thenounproject.com/icon/head-2288243/)” icon by Wildson Rafalski, “[Strawberry](https://thenounproject.com/icon/strawberry-3487721/)”icon by Ashabul Kahfi, “[Robot arm](https://thenounproject.com/icon/robot-arm-4466147/)” icon by Peter Lakenbrink, “[Lock](https://thenounproject.com/icon/lock-3107451/)” icon by IcoMoon and “[Stop](https://thenounproject.com/icon/stop-34715/)” icon by AS Design from Noun Project CC BY 3.0.

feedingwebapp/README.md

@@ -22,7 +22,8 @@ The overall user flow for this robot can be seen below.
 4. Source the directory: `source install/setup.bash`
 5. Navigate to the web app folder: `cd feeding_web_interface/feedingwebapp`
 6. Install web app dependencies: `npm install --legacy-peer-deps`
-* Consider checking out the [Troubleshooting](## Troubleshooting) section if there are errors in this process. 
+* Consider checking out the Troubleshooting section if there are errors in this process. 
+* Always run `source /opt/ros/humble/setup.bash` in the ROS2 workspace when a new terminal is opened.
 
 ### Usage (Web App)
 1. Navigate to the web app folder: `cd {path/to/feeding_web_interface}/feedingwebapp`

feedingwebapp/package.json

@@ -12,6 +12,7 @@
     "react": "^18.1.0",
     "react-bootstrap": "^2.3.1",
     "react-dom": "^18.1.0",
+    "react-hook-form": "^7.43.9",
     "react-native-web": "^0.19.4",
     "react-router-dom": "^6.3.0",
     "react-script-tag": "^1.1.2",

feedingwebapp/src/Pages/Constants.js

@@ -14,17 +14,31 @@ export const REALSENSE_HEIGHT = 480
 export const TIME_TO_RESET_MS = 3600000 // 1 hour in milliseconds
 
 /**
- * A dictionary containing the icon associated with each "robot motion" state.
+ * A dictionary containing the icon associated with each "robot moving" state
+ * except the Bite Acquisition state which has only a back button in footer.
+ * This dictionary is used to populate buttons with the icon images for states
+ * that those buttons will transition to.
  * The keys are the meal states where the robot moves.
  * Each key is paired with a value of an svg icon image of that meal state.
  */
-let FOOTER_STATE_ICON_DICT = {}
-FOOTER_STATE_ICON_DICT[MEAL_STATE.R_MovingAbovePlate] = '/robot_state_imgs/move_above_plate_position.svg'
-FOOTER_STATE_ICON_DICT[MEAL_STATE.R_BiteAcquisition] = '/robot_state_imgs/move_to_bite_acquisition_position.svg'
-FOOTER_STATE_ICON_DICT[MEAL_STATE.R_MovingToStagingLocation] = '/robot_state_imgs/move_to_staging_position.svg'
-FOOTER_STATE_ICON_DICT[MEAL_STATE.R_MovingToMouth] = '/robot_state_imgs/move_to_mouth_position.svg'
-FOOTER_STATE_ICON_DICT[MEAL_STATE.R_StowingArm] = '/robot_state_imgs/stowing_arm_position.svg'
-export { FOOTER_STATE_ICON_DICT }
+let MOVING_STATE_ICON_DICT = {}
+MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingAbovePlate] = '/robot_state_imgs/move_above_plate_position.svg'
+MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingToStagingLocation] = '/robot_state_imgs/move_to_staging_position.svg'
+MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingToMouth] = '/robot_state_imgs/move_to_mouth_position.svg'
+MOVING_STATE_ICON_DICT[MEAL_STATE.R_StowingArm] = '/robot_state_imgs/stowing_arm_position.svg'
+export { MOVING_STATE_ICON_DICT }
+
+/**
+ * A set containing the states where the robot does not move.
+ */
+let NON_MOVING_STATES = new Set()
+NON_MOVING_STATES.add(MEAL_STATE.U_PreMeal)
+NON_MOVING_STATES.add(MEAL_STATE.U_BiteSelection)
+NON_MOVING_STATES.add(MEAL_STATE.U_BiteAcquisitionCheck)
+NON_MOVING_STATES.add(MEAL_STATE.U_BiteInitiation)
+NON_MOVING_STATES.add(MEAL_STATE.U_BiteDone)
+NON_MOVING_STATES.add(MEAL_STATE.U_PostMeal)
+export { NON_MOVING_STATES }
 
 // The names of the ROS topic(s)
 export const CAMERA_FEED_TOPIC = '/camera/color/image_raw'

feedingwebapp/src/Pages/Footer/Footer.jsx

@@ -8,7 +8,7 @@ import Row from 'react-bootstrap/Row'
 import PropTypes from 'prop-types'
 
 // Local imports
-import { FOOTER_STATE_ICON_DICT } from '../Constants'
+import { MOVING_STATE_ICON_DICT } from '../Constants'
 import { useGlobalState } from '../GlobalState'
 
 /**
@@ -32,8 +32,8 @@ const Footer = (props) => {
 
   // Icons and other parameters for the footer buttons
   let pauseIcon = '/robot_state_imgs/pause_button_icon.svg'
-  let backIcon = props.backMealState ? FOOTER_STATE_ICON_DICT[props.backMealState] : ''
-  let resumeIcon = FOOTER_STATE_ICON_DICT[mealState]
+  let backIcon = props.backMealState ? MOVING_STATE_ICON_DICT[props.backMealState] : ''
+  let resumeIcon = MOVING_STATE_ICON_DICT[mealState]
   let phantomButtonIcon = '/robot_state_imgs/phantom_view_image.svg'
   // Width of Back and Resume buttons
   let backResumeButtonWidth = '150px'

feedingwebapp/src/Pages/GlobalState.jsx

@@ -87,12 +87,18 @@ export const SETTINGS = {
 export const useGlobalState = create(
   persist(
     (set) => ({
-      // Values stored in global state
+      // The app's current meal state
       mealState: MEAL_STATE.U_PreMeal,
+      // The timestamp when the robot transitioned to its current meal state
       mealStateTransitionTime: Date.now(),
+      // The current app page
       appPage: APP_PAGE.Home,
+      // The most recent food item that the user selected in "bite selection"
       desiredFoodItem: null,
+      // The center of the mouth as detected by face detection
       detectedMouthCenter: null,
+      // Whether or not the currently-executing robot motion was paused by the user
+      paused: false,
       // Settings values
       stagingPosition: SETTINGS.stagingPosition[0],
       biteInitiation: SETTINGS.biteInitiation[0],
@@ -116,6 +122,10 @@ export const useGlobalState = create(
         set(() => ({
           detectedMouthCenter: detectedMouthCenter
         })),
+      setPaused: (paused) =>
+        set(() => ({
+          paused: paused
+        })),
       setStagingPosition: (stagingPosition) =>
         set(() => ({
           stagingPosition: stagingPosition

feedingwebapp/src/Pages/Header/Header.jsx

@@ -3,6 +3,8 @@ import React, { useCallback, useEffect, useState } from 'react'
 // The NavBar is the navigation toolbar at the top
 import Navbar from 'react-bootstrap/Navbar'
 import Nav from 'react-bootstrap/Nav'
+// The Button is used for stop icon at the top
+import Button from 'react-bootstrap/Button'
 // PropTypes is used to validate that the used props are in fact passed to this
 // Component
 import PropTypes from 'prop-types'
@@ -13,7 +15,7 @@ import 'react-toastify/dist/ReactToastify.css'
 import { useROS } from '../../ros/ros_helpers'
 
 // Local imports
-import { ROS_CHECK_INTERVAL_MS } from '../Constants'
+import { ROS_CHECK_INTERVAL_MS, NON_MOVING_STATES } from '../Constants'
 import { useGlobalState, APP_PAGE, MEAL_STATE } from '../GlobalState'
 import LiveVideoModal from './LiveVideoModal'
 
@@ -44,6 +46,7 @@ const Header = (props) => {
   // Get the relevant global state variables
   const mealState = useGlobalState((state) => state.mealState)
   const setAppPage = useGlobalState((state) => state.setAppPage)
+  const paused = useGlobalState((state) => state.paused)
 
   /**
    * When the Home button in the header is clicked, return to the Home page.
@@ -79,7 +82,7 @@ const Header = (props) => {
        * of the robot is placed in between Settings and Video.
        */}
       <Navbar collapseOnSelect expand='lg' bg='dark' variant='dark' sticky='top'>
-        <Navbar id='responsive-navbar-nav'>
+        <Navbar id='responsive-navbar-nav' bg='dark' variant='dark'>
           <Nav className='me-auto'>
             <Nav.Link
               onClick={homeClicked}
@@ -96,15 +99,41 @@ const Header = (props) => {
               Settings
             </Nav.Link>
           </Nav>
+          {NON_MOVING_STATES.has(mealState) || paused ? (
+            <div>
+              <Button
+                variant='danger'
+                disabled={true}
+                style={{
+                  marginLeft: 3,
+                  marginRight: 3,
+                  width: '44px',
+                  height: '56px',
+                  opacity: 1,
+                  '--bs-btn-padding-y': '0rem',
+                  '--bs-btn-padding-x': '0rem'
+                }}
+              >
+                <img
+                  style={{ width: '44px', height: '50px' }}
+                  src='/robot_state_imgs/lock_icon_image.svg'
+                  alt='lock_icon_img'
+                  className='center'
+                />
+              </Button>
+            </div>
+          ) : (
+            <></>
+          )}
           {isConnected ? (
             <div>
-              <p className='connectedDiv' style={{ fontSize: '24px' }}>
+              <p className='connectedDiv' style={{ fontSize: '24px', margin: 3 }}>
                 🔌
               </p>
             </div>
           ) : (
             <div>
-              <p className='notConnectedDiv' style={{ fontSize: '24px' }}>
+              <p className='notConnectedDiv' style={{ fontSize: '24px', marginLeft: 3, marginRight: 3 }}>
                 ⛔
               </p>
             </div>

feedingwebapp/src/Pages/Home/Home.jsx

@@ -29,6 +29,7 @@ function Home(props) {
   const mealState = useGlobalState((state) => state.mealState)
   const mealStateTransitionTime = useGlobalState((state) => state.mealStateTransitionTime)
   const setMealState = useGlobalState((state) => state.setMealState)
+  const setPaused = useGlobalState((state) => state.setPaused)
 
   /**
    * Implement time-based transition of states. This is so that after the user
@@ -40,8 +41,9 @@ function Home(props) {
     if (Date.now() - mealStateTransitionTime >= TIME_TO_RESET_MS) {
       console.log('Reverting to PreMeal due to too much elapsed time in one state.')
       setMealState(MEAL_STATE.U_PreMeal)
+      setPaused(false)
     }
-  }, [mealStateTransitionTime, setMealState])
+  }, [mealStateTransitionTime, setMealState, setPaused])
 
   // Get the relevant global variables
   const desiredFoodItem = useGlobalState((state) => state.desiredFoodItem)