[Core][Python3.10] Refactor codebase for Python 3.10 compatibility, r…

…emoving Python 2.7 related stuff (#3663)

Co-authored-by: agl29 <ayush.goyal@ayushlocal>

Makefile

@@ -78,8 +78,6 @@
 ###################################
 ROOT := $(realpath .)
 PPC64LE := $(shell uname -m)
-VIRTUAL_ENV_VERSION := 20.24.4
-VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
 
 include $(ROOT)/Makefile.vars.priv
 
@@ -133,30 +131,11 @@ endif
 ###################################
 
 .PHONY: virtual-env
-virtual-env: virtual-env2.7 virtual-env3.8 virtual-env3.9
-
-.PHONY: virtual-env2.7
-virtual-env2.7: $(BLD_DIR_ENV)/stamp
-$(BLD_DIR_ENV)/stamp:
-ifeq ($(PYTHON_VER),python2.7)
-	@echo "--- Creating virtual environment at $(BLD_DIR_ENV) using $(PYTHON_VER)"
-	@$(SYS_PYTHON) $(VIRTUAL_BOOTSTRAP) $(VIRTUALENV_OPTS) --system-site-packages $(BLD_DIR_ENV)
-	@echo "--- Virtual environment $(BLD_DIR_ENV) ready"
-	@touch $@
-	@echo '--- Installing PIP_MODULES in virtual-env'
-	@echo "--- start installing PIP_MODULES in virtual-env"
-	@$(ENV_PIP) install --upgrade pip
-	@$(ENV_PIP) install --upgrade --force-reinstall $(PIP_MODULES)
-	@echo "--- done installing PIP_MODULES in virtual-env"
-endif
-
-.PHONY: virtual-env3.8
-virtual-env3.8: $(BLD_DIR_ENV)/stamp
+virtual-env: $(BLD_DIR_ENV)/stamp
 $(BLD_DIR_ENV)/stamp:
-ifeq ($(PYTHON_VER),python3.8)
 	@echo "--- Creating virtual environment at $(BLD_DIR_ENV) using $(PYTHON_VER)"
-	@$(SYS_PYTHON) -m pip install --upgrade pip==22.2.2
-	@$(SYS_PIP) install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
+	@$(SYS_PYTHON) -m pip install --upgrade pip==$(PIP_VERSION)
+	$(SYS_PIP) install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
 	@if [[ "ppc64le" == $(PPC64LE) ]]; then \
 	  $(SYS_PYTHON) -m venv $(BLD_DIR_ENV); \
 	 fi
@@ -176,35 +155,6 @@ ifeq ($(PYTHON_VER),python3.8)
 	@$(ENV_PIP) install $(NAVOPTAPI_WHL)
 	@echo '--- Finished $(NAVOPTAPI_WHL) into virtual-env'
 	@touch $(REQUIREMENT_DOT_FILE)
-endif
-
-.PHONY: virtual-env3.9
-virtual-env3.9: $(BLD_DIR_ENV)/stamp
-$(BLD_DIR_ENV)/stamp:
-ifeq ($(PYTHON_VER),python3.9)
-	@echo "--- Creating virtual environment at $(BLD_DIR_ENV) using $(PYTHON_VER)"
-	@$(SYS_PYTHON) -m pip install --upgrade pip==22.2.2
-	@$(SYS_PIP) install virtualenv==20.19.0 virtualenv-make-relocatable==0.0.1
-	@if [[ "ppc64le" == $(PPC64LE) ]]; then \
-	  $(SYS_PYTHON) -m venv $(BLD_DIR_ENV); \
-	 fi
-	@virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV)
-	@echo "--- Virtual environment $(BLD_DIR_ENV) ready"
-	@touch $@
-	@echo '--- Installing PIP_MODULES in virtual-env'
-	@if [[ "ppc64le" == $(PPC64LE) ]]; then \
-	  echo '--- Installing $(REQUIREMENT_PPC64LE_FILE_PY39) into virtual-env via $(ENV_PIP)'; \
-	  $(ENV_PIP) install -r $(REQUIREMENT_PPC64LE_FILE_PY39); \
-	  echo '--- Finished $(REQUIREMENT_PPC64LE_FILE_PY39) into virtual-env'; \
-	 else \
-	  echo '--- Installing $(REQUIREMENT_FILE) into virtual-env via $(ENV_PIP)'; \
-	  $(ENV_PIP) install -r $(REQUIREMENT_FILE); \
-	  echo '--- Finished $(REQUIREMENT_FILE) into virtual-env'; \
-         fi
-	@$(ENV_PIP) install $(NAVOPTAPI_WHL)
-	@echo '--- Finished $(NAVOPTAPI_WHL) into virtual-env'
-	@touch $(REQUIREMENT_DOT_FILE)
-endif
 ###################################
 # Build desktop
 ###################################
@@ -234,7 +184,6 @@ INSTALL_CORE_FILES = \
 	ext \
 	tools/app_reg \
 	tools/virtual-bootstrap \
-	tools/enable-python27.sh \
 	tools/relocatable.sh \
 	VERS* LICENSE* README*
 

Makefile.sdk

@@ -131,12 +131,6 @@ $(REQUIREMENT_DOT_FILE):
 .PHONY: ext-eggs
 ext-eggs: $(EXT_PY_EGG_TARGETS)
 $(EXT_PY_EGG_TARGETS):
-ifeq ($(PYTHON_VER),python2.7)
-	@echo '--- Building egg for $(CUR_EXT_PY)'
-	@cd ext-py/$(CUR_EXT_PY) && $(ENV_PYTHON) \
-	  -c '__import__("setuptools.sandbox").sandbox.run_setup("setup.py", __import__("sys").argv[1:])' \
-	  bdist_egg
-endif
 	@mkdir -p $(@D) && touch $@
 
 #

Makefile.tarball

@@ -29,7 +29,6 @@ PROD_INCLUDES := \
 	tools/app_reg \
 	tools/virtual-bootstrap \
 	tools/relocatable.sh \
-	tools/enable-python27.sh \
 	tools/load-balancer \
 	VERSION \
 	webpack-stats*.json \

Makefile.vars

@@ -37,21 +37,17 @@ ifneq ($(wildcard $(ROOT)/Makefile.buildvars),)
 endif
 
 
-VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports python2.7 or >= python3.8. If not set, defaults to python2.7."
+VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports >= python3.8. If not set, defaults to python3.8."
 
-PYTHON_VER ?= python2.7
+PYTHON_VER ?= python3.8
 PYTHON_EXE = $(shell echo $(PYTHON_VER) | sed "s/\.//")
-SYS_PIP = "pip2.7"
+SYS_PIP = "pip3.8"
 
 $(info "PYTHON_VER is $(PYTHON_VER).")
 
-ifeq ($(PYTHON_VER),python2.7)
-  $(info "Python 2 module install of desktop/ext-py")
-  EXT_ENV_INSTALL = ext-env-install
-  SYS_PIP := $(shell which pip2.7)
-else ifeq ($(shell echo $(PYTHON_VER) | head -c 8),python3.)
-  PYTHON_VER := $(shell echo $(PYTHON_VER) | head -c 9)
-  MINOR_VER = $(shell echo -n $(PYTHON_VER) | tail -c 1)
+ifeq ($(shell echo $(PYTHON_VER) | head -c 8),python3.)
+  PYTHON_VER := $(shell echo $(PYTHON_VER) | sed 's/\(python3\.[0-9]*\).*/\1/')
+  MINOR_VER = $(shell echo $(PYTHON_VER) | sed 's/python3\.//')
   ifeq ($(shell test $(MINOR_VER) -lt 8; echo $$?),0)
     $(error "$(VER_ERR_MSG)")
   endif
@@ -72,7 +68,7 @@ else
 endif
 
 ifeq ($(SYS_PYTHON),)
-  $(error "Error: Need python version 2.7 or >= 3.5")
+  $(error "Error: Need python version >= 3.8")
 else
   $(info "SYS_PYTHON is $(SYS_PYTHON).")
 endif
@@ -143,7 +139,24 @@ PYBABEL := $(ROOT)/build/env/bin/pybabel
 # Path to files for pip requirements
 ##############################
 REQUIREMENT_FILE := $(ROOT)/desktop/core/requirements.txt
-REQUIREMENT_PPC64LE_FILE := $(ROOT)/desktop/core/requirements_ppc64le.txt
-REQUIREMENT_PPC64LE_FILE_PY39 := $(ROOT)/desktop/core/requirements_ppc64le_py39.txt
 REQUIREMENT_DOT_FILE := $(ROOT)/desktop/core/.requirements
 NAVOPTAPI_WHL := $(ROOT)/desktop/core/wheels/navoptapi-1.0.0-py3-none-any.whl
+
+##############################
+# Set specific versions for some libraries
+##############################
+ifeq ($(PYTHON_VER),python3.8)
+  PIP_VERSION := 22.2.2
+  VIRTUAL_ENV_VERSION := 20.24.4
+  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
+  REQUIREMENT_PPC64LE_FILE := $(ROOT)/desktop/core/requirements_ppc64le.txt
+else ifeq ($(PYTHON_VER),python3.9)
+  PIP_VERSION := 22.2.2
+  VIRTUAL_ENV_VERSION := 20.19.0
+  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
+  REQUIREMENT_PPC64LE_FILE := $(ROOT)/desktop/core/requirements_ppc64le_py39.txt
+else ifeq ($(PYTHON_VER),python3.10)
+  PIP_VERSION := 22.2.2
+  VIRTUAL_ENV_VERSION := 20.24.4
+  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
+endif

desktop/devtools.mk

@@ -31,11 +31,6 @@ PYPI_MIRROR ?= https://pypi.python.org/simple/
 # Install/download dev tools for SDK into the virtual environment
 .PHONY: $(DEVTOOLS)
 $(DEVTOOLS):
-ifeq ($(PYTHON_VER),python2.7)
-	@echo "--- Installing development tool: $@"
-	$(ENV_PIP)  \
-	   install $(subst ],,$(subst [,==,$@))
-endif
 	@echo
 
 

tools/relocatable.sh

@@ -93,13 +93,6 @@ if [[ $PYVER == $MIN_VERSION || $PYVER == $MIN_VERSION.* || $PYVER > $MIN_VERSIO
   virtualenv-make-relocatable "build/env"
   $ENV_PYTHON $VIRTUAL_BOOTSTRAP --relocatable_pth "build/env"
   popd
-else
-  echo "Python version is less than 3.8"
-  if [ -e "$HUE_ROOT/tools/enable-python27.sh" ]; then
-    source $HUE_ROOT/tools/enable-python27.sh
-  fi
-
-  $ENV_PYTHON $VIRTUAL_BOOTSTRAP --relocatable "$HUE_ROOT/build/env"
 fi
 
 # Step 1b. Fix any broken lib64 directory