feat: ansible upgrade

Author: nadeemshahzad

Makefile

@@ -20,6 +20,11 @@ requirements:
 	pip install -qr requirements/pip.txt --exists-action w
 	pip install -qr requirements.txt --exists-action w
 
+requirements3_12:
+	pip install -qr requirements/pip.txt --exists-action w
+	pip install -qr requirements3_12txt --exists-action w
+
+
 COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
 .PHONY: $(COMMON_CONSTRAINTS_TXT)
 $(COMMON_CONSTRAINTS_TXT):

playbooks/library/ec2_lookup

@@ -16,7 +16,10 @@
 
 from __future__ import absolute_import
 from __future__ import print_function
-import six
+import sys
+import boto3
+from ansible.module_utils.basic import AnsibleModule
+
 DOCUMENTATION = '''
 ---
 module: ec2_lookup
@@ -28,8 +31,8 @@ options:
   region:
     description:
       - The AWS region to use.  Must be specified if ec2_url
-        is not used.  If not specified then the value of the
-        EC2_REGION environment variable, if any, is used.
+        is not used.  If not specified then the value of
+        the EC2_REGION environment variable, if any, is used.
     required: false
     default: null
     aliases: [ 'aws_region', 'ec2_region' ]
@@ -42,20 +45,20 @@ options:
     aliases: [ 'ec2_secret_key', 'secret_key' ]
   aws_access_key:
     description:
-      - AWS access key. If not set then the value of the
-        AWS_ACCESS_KEY environment variable is used.
+      - AWS access key. If not set then the value of
+        the AWS_ACCESS_KEY environment variable is used.
     required: false
     default: null
     aliases: [ 'ec2_access_key', 'access_key' ]
   tags:
-    desription:
+    description:
       - tags to lookup
     required: false
     default: null
     type: dict
     aliases: []
 
-requirements: [ "boto" ]
+requirements: [ "boto3" ]
 author: John Jarvis
 '''
 
@@ -70,8 +73,6 @@ EXAMPLES = '''
         Name: foo
 '''
 
-import sys
-
 AWS_REGIONS = ['ap-northeast-1',
                'ap-southeast-1',
                'ap-southeast-2',
@@ -81,17 +82,8 @@ AWS_REGIONS = ['ap-northeast-1',
                'us-west-1',
                'us-west-2']
 
-try:
-    import boto.ec2
-    from boto.ec2 import connect_to_region
-except ImportError:
-    print("failed=True msg='boto required for this module'")
-    sys.exit(1)
-
-
 def main():
-
-    module=AnsibleModule(
+    module = AnsibleModule(
         argument_spec=dict(
             ec2_url=dict(),
             region=dict(aliases=['aws_region', 'ec2_region'],
@@ -108,35 +100,34 @@ def main():
     region = module.params.get('region')
     ec2_url = module.params.get('ec2_url')
 
-    # If we have a region specified, connect to its endpoint.
+    # If region is specified, create a session with boto3 and connect to the EC2 service
     if region:
         try:
-            ec2 = connect_to_region(region, aws_access_key_id=aws_access_key,
-                                    aws_secret_access_key=aws_secret_key)
-        except boto.exception.NoAuthHandlerFound as e:
-            module.fail_json(msg=str(e))
-    # If we specified an ec2_url then try connecting to it
-    elif ec2_url:
-        try:
-            ec2 = boto.connect_ec2_endpoint(ec2_url, aws_access_key,
-                                            aws_secret_key)
-        except boto.exception.NoAuthHandlerFound as e:
-            module.fail_json(msg=str(e))
+            ec2 = boto3.resource('ec2', region_name=region, aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key)
+        except Exception as e:
+            module.fail_json(msg=f"Error connecting to AWS EC2: {str(e)}")
     else:
-        module.fail_json(msg="Either region or ec2_url must be specified")
+        module.fail_json(msg="Region must be specified")
+
+    # If EC2 URL is provided, it can be used for connection
+    if ec2_url:
+        module.fail_json(msg="ec2_url is not supported in boto3 connection. Please use region instead.")
 
     instances = []
     instance_ids = []
-    for res in ec2.get_all_instances(filters={'tag:' + tag: value
-                                     for tag, value in six.iteritems(module.params.get('tags'))}):
-        for inst in res.instances:
-            if inst.state == "running":
-                instances.append({k: v for k, v in six.iteritems(inst.__dict__)
-                                  if isinstance(v, (six.string_types))})
-                instance_ids.append(inst.id)
-    module.exit_json(changed=False, instances=instances,
-                     instance_ids=instance_ids)
 
+    # Get all instances with the specified tags
+    filters = {'tag:' + tag: value for tag, value in module.params.get('tags', {}).items()}
+    try:
+        instances_query = ec2.instances.filter(Filters=[{'Name': f'tag:{tag}', 'Values': [value]} for tag, value in filters.items()])
+        for instance in instances_query:
+            if instance.state['Name'] == 'running':
+                instances.append({k: v for k, v in instance.__dict__.items() if isinstance(v, str)})
+                instance_ids.append(instance.id)
+    except Exception as e:
+        module.fail_json(msg=f"Error querying EC2 instances: {str(e)}")
+
+    module.exit_json(changed=False, instances=instances, instance_ids=instance_ids)
 
 # this is magic, see lib/ansible/module_common.py
 #<<INCLUDE_ANSIBLE_MODULE_COMMON>>

playbooks/roles/launch_ec2/tasks/main.yml

@@ -35,26 +35,27 @@
   when: terminate_instance == true and elb and tag_lookup.instance_ids|length == 1
 
 - name: Launch ec2 instance
-  local_action:
-    module: ec2
-    keypair: "{{ keypair }}"
-    group: "{{ security_group }}"
+  community.aws.ec2_instance:
+    key_name: "{{ keypair }}"
+    security_groups: 
+     - "{{ security_group }}"
     instance_type: "{{ instance_type }}"
     instance_initiated_shutdown_behavior: "{{ instance_initiated_shutdown_behavior }}"
-    image: "{{ ami }}"
+    image_id: "{{ ami }}"
     vpc_subnet_id: "{{ vpc_subnet_id }}"
-    assign_public_ip: yes
     wait: true
     region: "{{ region }}"
-    instance_tags: "{{ instance_tags }}"
+    tags: "{{ instance_tags }}"
+    network:
+      assign_public_ip: true
     volumes:
       - device_name: /dev/sda1
-        volume_size: "{{ root_ebs_size }}"
-        delete_on_termination: true
-        volume_type: "gp2"
-        encrypted: true
-    zone: "{{ zone }}"
-    instance_profile_name: "{{ instance_profile_name }}"
+        ebs:
+          volume_size: "{{ root_ebs_size }}"
+          delete_on_termination: true
+          volume_type: "gp2"
+          encrypted: true
+    instance_role: "{{ instance_profile_name }}"
     user_data: "{{ user_data }}"
   register: ec2
 
@@ -97,7 +98,7 @@
 - name: Add new instance to host group
   local_action:
     module: add_host
-    hostname: "{{ item.public_ip }}"
+    hostname: "{{ item.network_interfaces[0].association.public_ip }}"
     groups: launched
   with_items: "{{ ec2.instances }}"
 

util/install/ansible-bootstrap.sh

@@ -33,7 +33,9 @@ fi
 # Bootstrapping constants
 #
 VIRTUAL_ENV_VERSION="16.7.10"
+VIRTUAL_ENV_VERSION_NOBLE="20.27.0"
 PIP_VERSION="21.2.1"
+PIP_VERSION_NOBLE="24.0"
 SETUPTOOLS_VERSION="44.1.0"
 VIRTUAL_ENV="/tmp/bootstrap"
 PYTHON_BIN="${VIRTUAL_ENV}/bin"
@@ -71,6 +73,9 @@ then
 elif grep -q 'Focal Fossa' /etc/os-release
 then
     SHORT_DIST="focal"
+elif grep -q 'Noble Numbat' /etc/os-release
+then
+    SHORT_DIST="noble"
 else
     cat << EOF
 
@@ -82,7 +87,9 @@ EOF
    exit 1;
 fi
 
-if [[ "${SHORT_DIST}" == focal ]] ;then
+if [[ "${SHORT_DIST}" == noble ]]; then
+   PYTHON_VERSION="3.12" 
+elif [[ "${SHORT_DIST}" == focal ]] ;then
    PYTHON_VERSION="3.8"
 else
    PYTHON_VERSION="3.5"
@@ -108,7 +115,7 @@ fi
 
 # Required for add-apt-repository
 apt-get install -y software-properties-common
-if [[ "${SHORT_DIST}" != trusty ]] && [[ "${SHORT_DIST}" != xenial ]] && [[ "${SHORT_DIST}" != bionic ]] && [[ "${SHORT_DIST}" != focal ]] ;then
+if [[ "${SHORT_DIST}" != trusty ]] && [[ "${SHORT_DIST}" != xenial ]] && [[ "${SHORT_DIST}" != bionic ]] && [[ "${SHORT_DIST}" != focal ]] && [[ "${SHORT_DIST}" != noble ]] ;then
   apt-get install -y python-software-properties
 fi
 
@@ -117,7 +124,7 @@ add-apt-repository -y ppa:git-core/ppa
 
 # For older software we need to install our own PPA
 # Phased out with Ubuntu 18.04 Bionic and Ubuntu 20.04 Focal
-if [[ "${SHORT_DIST}" != bionic ]] && [[ "${SHORT_DIST}" != focal ]] ;then
+if [[ "${SHORT_DIST}" != bionic ]] && [[ "${SHORT_DIST}" != focal ]] && [[ "${SHORT_DIST}" != noble ]] ;then
   apt-key adv --keyserver "${EDX_PPA_KEY_SERVER}" --recv-keys "${EDX_PPA_KEY_ID}"
   add-apt-repository -y "${EDX_PPA}"
 fi
@@ -133,7 +140,7 @@ fi
 # which may differ from what is pinned in virtualenvironments
 apt-get update -y
 
-if [[ "${SHORT_DIST}" != focal ]] ;then
+if [[ "${SHORT_DIST}" != focal ]] && [[ "${SHORT_DIST}" != noble ]] ;then
   apt-get install -y python2.7 python2.7-dev python-pip python-apt python-jinja2 build-essential sudo git-core libmysqlclient-dev libffi-dev libssl-dev
 else
   apt-get install -y python3-pip python3-apt python3-jinja2 build-essential sudo git-core libmysqlclient-dev libffi-dev libssl-dev
@@ -143,17 +150,24 @@ apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-p
 
 # We want to link pip to pip3 for Ubuntu versions that don't have python 2.7 so older scripts work there
 # Applies to Ubuntu 20.04 Focal
-if [[ "${SHORT_DIST}" != trusty ]] && [[ "${SHORT_DIST}" != xenial ]] && [[ "${SHORT_DIST}" != bionic ]] ;then
+if [[ "${SHORT_DIST}" != trusty ]] && [[ "${SHORT_DIST}" != xenial ]] && [[ "${SHORT_DIST}" != bionic ]] && [[ "${SHORT_DIST}" != noble ]] ;then
   sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1
   sudo update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1
 fi
 
+if [[ "${SHORT_DIST}" == noble ]] ;then
+  sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1
+  sudo update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1
+fi
+
+if [[ "${SHORT_DIST}" != noble ]] ;then
 python${PYTHON_VERSION} -m pip install --upgrade pip=="${PIP_VERSION}"
 
 # pip moves to /usr/local/bin when upgraded
 PATH=/usr/local/bin:${PATH}
 python${PYTHON_VERSION} -m pip install setuptools=="${SETUPTOOLS_VERSION}"
 python${PYTHON_VERSION} -m pip install virtualenv=="${VIRTUAL_ENV_VERSION}"
+fi
 
 if [[ "true" == "${RUN_ANSIBLE}" ]]; then
     # create a new virtual env
@@ -166,7 +180,11 @@ if [[ "true" == "${RUN_ANSIBLE}" ]]; then
     git clone ${CONFIGURATION_REPO} ${CONFIGURATION_DIR}
     cd ${CONFIGURATION_DIR}
     git checkout ${CONFIGURATION_VERSION}
+    if [[ "${SHORT_DIST}" != noble ]] ;then
+    make requirements3_12
+    else
     make requirements
+    fi
 
     cd "${CONFIGURATION_DIR}"/playbooks
     "${PYTHON_BIN}"/ansible-playbook edx_ansible.yml -i '127.0.0.1,' -c local -e "CONFIGURATION_VERSION=${CONFIGURATION_VERSION}"

util/jenkins/ansible-provision.sh

@@ -172,11 +172,10 @@ if [[ -z $ami ]]; then
     ami="ami-089b5711e63812c2a"
     # Ansible will always use Python3 interpreter on Ubuntu 20.04 hosts to execute modules
     extra_var_arg+=' -e ansible_python_interpreter=auto'
-  fi
   elif [[ $server_type == "ubuntu_24.04" ]]; then
     ami="ami-04b4f1a9cf54c11d0"
-    # Ansible will always use Python3 interpreter on Ubuntu 24.04 hosts to execute modules
-    extra_var_arg+=' -e ansible_python_interpreter=auto'
+    # Ansible will always use Python3.12 interpreter on Ubuntu 24.04 hosts to execute modules
+    extra_var_arg+=' -e ansible_python_interpreter=/usr/bin/python3.12'
   fi
 fi