Skip to content
Snippets Groups Projects
Commit 09c25bf3 authored by Sebastian Rieger's avatar Sebastian Rieger
Browse files

changed AWS instance size to t2.micro, t2.nano is still possible using... 

changed AWS instance size to t2.micro, t2.nano is still possible using install-aws.sh cloud-init script, but really limited in RAM, added support for availability zones and spread instance across different az to improve fault tolerance, select AMI directly, remove 0.0.0.0/0 access to mysql
parent e18ab4dc
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
example-projects/demo4-scale-out-lb-in-aws-destroy.py
+ 6
− 2
View file @ 09c25bf3
...@@ -59,9 +59,10 @@ def main(): ...@@ -59,9 +59,10 @@ def main():
token=aws_session_token, token=aws_session_token,
region=region_name) region=region_name)
print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers())) # print("List of load balancers: " + str(elb_conn.list_balancers()))
for loadbalancer in elb_conn.list_balancers(): for loadbalancer in elb_conn.list_balancers():
if loadbalancer.name == "lb1": if loadbalancer.name == "lb1":
print("Deleting Load Balancer" + str(loadbalancer))
elb_conn.destroy_balancer(loadbalancer) elb_conn.destroy_balancer(loadbalancer)
########################################################################### ###########################################################################
...@@ -101,7 +102,10 @@ def main(): ...@@ -101,7 +102,10 @@ def main():
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-controller', 'app-services']: if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-controller', 'app-services']:
if instance.state is not NodeState.TERMINATED: if instance.state is not NodeState.TERMINATED:
nodes_still_running = True nodes_still_running = True
if nodes_still_running is True:
print('There are still instances running, waiting for them to be destroyed...') print('There are still instances running, waiting for them to be destroyed...')
else:
print('No instances running')
# delete security groups # delete security groups
for group in conn.ex_list_security_groups(): for group in conn.ex_list_security_groups():
......
example-projects/demo4-scale-out-lb-in-aws.py
+ 56
− 23
View file @ 09c25bf3
...@@ -22,6 +22,7 @@ home = expanduser("~") ...@@ -22,6 +22,7 @@ home = expanduser("~")
# The image to look for and use for the started instance # The image to look for and use for the started instance
ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128' ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
# ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
# TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3... # TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3...
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
...@@ -32,14 +33,16 @@ ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-2021 ...@@ -32,14 +33,16 @@ ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-2021
keypair_name = 'srieger-pub' keypair_name = 'srieger-pub'
pub_key_file = home + '/.ssh/id_rsa.pub' pub_key_file = home + '/.ssh/id_rsa.pub'
flavor_name = 't2.nano' # flavor_name = 't2.nano'
flavor_name = 't2.micro'
# default region # default region
# region_name = 'eu-central-1' # region_name = 'eu-central-1'
# region_name = 'ap-south-1' # region_name = 'ap-south-1'
# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/ # AWS Academy Labs only allow us-east-1 and us-west-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
region_name = 'us-east-1' region_name = 'us-east-1'
# region_name = 'us-west-1'
# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout # starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout
timeout = 600 timeout = 600
...@@ -84,9 +87,9 @@ def main(): ...@@ -84,9 +87,9 @@ def main():
# #
########################################################################### ###########################################################################
print("Search for AMI...") # print("Search for AMI...")
image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0] # image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0]
print("Using image: %s" % image) # print("Using image: %s" % image)
# print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of " # print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of "
# "supported operating systems and versions)... please be patient...") # "supported operating systems and versions)... please be patient...")
...@@ -98,18 +101,27 @@ def main(): ...@@ -98,18 +101,27 @@ def main():
# if img.id == ubuntu_image_name: # if img.id == ubuntu_image_name:
# image = img # image = img
# fetch/select the image referenced with ubuntu_image_name above
# image = [i for i in images if i.name == ubuntu_image_name][0]
# print(image)
# select image directly to save time, as retrieving the image list takes several minutes now, # select image directly to save time, as retrieving the image list takes several minutes now,
# need to change ami id here if updated or for other regions, id is working for course in # need to change ami id here if updated or for other regions, id is working for course in
# summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard, # summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard,
# to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the # to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the
# desired region # desired region
# image = NodeImage(id="ami-0e472ba40eb589f49", #
# name=ubuntu_image_name,
# driver="hvm") print("Selecting AMI...")
# us-east-1 examples as of 9.5.2022:
#
# Canonical, Ubuntu, 18.04 LTS, amd64 bionic image build on 2022-04-11
image_id = "ami-005de95e8ff495156"
#
# Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on 2022-04-19
# image_id = "ami-0c4f7023847b90238"
#
# Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on 2022-04-20
# image_id = "ami-09d56f8956ab235b3"
#
image = conn.list_images(ex_image_ids=[image_id])[0]
print("Using image: %s" % image)
flavors = conn.list_sizes() flavors = conn.list_sizes()
flavor = [s for s in flavors if s.id == flavor_name][0] flavor = [s for s in flavors if s.id == flavor_name][0]
...@@ -164,10 +176,14 @@ def main(): ...@@ -164,10 +176,14 @@ def main():
instances = conn.list_nodes() instances = conn.list_nodes()
for instance in instances: for instance in instances:
# if we see any demo instances still running continue to wait for them to stop # if we see any demo instances still running continue to wait for them to stop
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-controller', 'app-services']: if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller',
'app-services', 'app-api-1', 'app-api-2']:
if instance.state is not NodeState.TERMINATED: if instance.state is not NodeState.TERMINATED:
nodes_still_running = True nodes_still_running = True
if nodes_still_running is True:
print('There are still instances running, waiting for them to be destroyed...') print('There are still instances running, waiting for them to be destroyed...')
else:
print('No instances running')
# delete security groups, respecting dependencies (hence deleting 'control' and 'services' first) # delete security groups, respecting dependencies (hence deleting 'control' and 'services' first)
for group in conn.ex_list_security_groups(): for group in conn.ex_list_security_groups():
...@@ -233,7 +249,9 @@ def main(): ...@@ -233,7 +249,9 @@ def main():
services_security_group_id = services_security_group_result['group_id'] services_security_group_id = services_security_group_result['group_id']
conn.ex_authorize_security_group_ingress(services_security_group_id, 22, 22, cidr_ips=['0.0.0.0/0'], conn.ex_authorize_security_group_ingress(services_security_group_id, 22, 22, cidr_ips=['0.0.0.0/0'],
protocol='tcp') protocol='tcp')
conn.ex_authorize_security_group_ingress(services_security_group_id, 3306, 3306, cidr_ips=['0.0.0.0/0'], # conn.ex_authorize_security_group_ingress(services_security_group_id, 3306, 3306, cidr_ips=['0.0.0.0/0'],
# group_pairs=[{'group_id': api_security_group_id}], protocol='tcp')
conn.ex_authorize_security_group_ingress(services_security_group_id, 3306, 3306,
group_pairs=[{'group_id': api_security_group_id}], protocol='tcp') group_pairs=[{'group_id': api_security_group_id}], protocol='tcp')
conn.ex_authorize_security_group_ingress(services_security_group_id, 5672, 5672, conn.ex_authorize_security_group_ingress(services_security_group_id, 5672, 5672,
group_pairs=[{'group_id': worker_security_group_id}], protocol='tcp') group_pairs=[{'group_id': worker_security_group_id}], protocol='tcp')
...@@ -245,6 +263,12 @@ def main(): ...@@ -245,6 +263,12 @@ def main():
for security_group in conn.ex_list_security_groups(): for security_group in conn.ex_list_security_groups():
print(security_group) print(security_group)
# get availability zones
az = conn.list_locations()
print(az)
########################################################################### ###########################################################################
# #
# create app-services instance (database & messaging) (Amazon AWS EC2) # create app-services instance (database & messaging) (Amazon AWS EC2)
...@@ -258,7 +282,7 @@ def main(): ...@@ -258,7 +282,7 @@ def main():
# Thanks to Stefan Friedmann for finding this fix ;) # Thanks to Stefan Friedmann for finding this fix ;)
userdata_service = '''#!/usr/bin/env bash userdata_service = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \ curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \
-i database -i messaging -i database -i messaging
rabbitmqctl add_user faafo guest rabbitmqctl add_user faafo guest
rabbitmqctl set_user_tags faafo administrator rabbitmqctl set_user_tags faafo administrator
...@@ -266,7 +290,8 @@ def main(): ...@@ -266,7 +290,8 @@ def main():
''' '''
print('Starting new app-services instance and wait until it is running...') print('Starting new app-services instance and wait until it is running...')
instance_services = conn.create_node(name='app-services', instance_services = conn.create_node(location=az[0],
name='app-services',
image=image, image=image,
size=flavor, size=flavor,
ex_keyname=keypair_name, ex_keyname=keypair_name,
...@@ -282,13 +307,14 @@ def main(): ...@@ -282,13 +307,14 @@ def main():
########################################################################### ###########################################################################
userdata_api = '''#!/usr/bin/env bash userdata_api = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \ curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \
-i faafo -r api -m 'amqp://faafo:guest@%(services_ip)s:5672/' \ -i faafo -r api -m 'amqp://faafo:guest@%(services_ip)s:5672/' \
-d 'mysql+pymysql://faafo:password@%(services_ip)s:3306/faafo' -d 'mysql+pymysql://faafo:password@%(services_ip)s:3306/faafo'
''' % {'services_ip': services_ip} ''' % {'services_ip': services_ip}
print('Starting new app-api-1 instance and wait until it is running...') print('Starting new app-api-1 instance and wait until it is running...')
instance_api_1 = conn.create_node(name='app-api-1', instance_api_1 = conn.create_node(location=az[0],
name='app-api-1',
image=image, image=image,
size=flavor, size=flavor,
ex_keyname=keypair_name, ex_keyname=keypair_name,
...@@ -296,7 +322,8 @@ def main(): ...@@ -296,7 +322,8 @@ def main():
ex_security_groups=["api"]) ex_security_groups=["api"])
print('Starting new app-api-2 instance and wait until it is running...') print('Starting new app-api-2 instance and wait until it is running...')
instance_api_2 = conn.create_node(name='app-api-2', instance_api_2 = conn.create_node(location=az[1],
name='app-api-2',
image=image, image=image,
size=flavor, size=flavor,
ex_keyname=keypair_name, ex_keyname=keypair_name,
...@@ -305,8 +332,11 @@ def main(): ...@@ -305,8 +332,11 @@ def main():
instance_api_1 = conn.wait_until_running(nodes=[instance_api_1], timeout=120, ssh_interface='public_ips') instance_api_1 = conn.wait_until_running(nodes=[instance_api_1], timeout=120, ssh_interface='public_ips')
api_1_ip = instance_api_1[0][0].private_ips[0] api_1_ip = instance_api_1[0][0].private_ips[0]
print("app-api-1 public ip: " + instance_api_1[0][1][0])
instance_api_2 = conn.wait_until_running(nodes=[instance_api_2], timeout=120, ssh_interface='public_ips') instance_api_2 = conn.wait_until_running(nodes=[instance_api_2], timeout=120, ssh_interface='public_ips')
# currently only api_1_ip is used
api_2_ip = instance_api_2[0][0].private_ips[0] api_2_ip = instance_api_2[0][0].private_ips[0]
print("app-api-2 public ip: " + instance_api_2[0][1][0])
########################################################################### ###########################################################################
# #
...@@ -315,7 +345,7 @@ def main(): ...@@ -315,7 +345,7 @@ def main():
########################################################################### ###########################################################################
userdata_worker = '''#!/usr/bin/env bash userdata_worker = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \ curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \
-i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/' -i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/'
''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip} ''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip}
...@@ -325,14 +355,16 @@ def main(): ...@@ -325,14 +355,16 @@ def main():
# ''' % {'api_2_ip': api_2_ip, 'services_ip': services_ip} # ''' % {'api_2_ip': api_2_ip, 'services_ip': services_ip}
print('Starting new app-worker-1 instance and wait until it is running...') print('Starting new app-worker-1 instance and wait until it is running...')
instance_worker_1 = conn.create_node(name='app-worker-1', instance_worker_1 = conn.create_node(location=az[0],
name='app-worker-1',
image=image, size=flavor, image=image, size=flavor,
ex_keyname=keypair_name, ex_keyname=keypair_name,
ex_userdata=userdata_worker, ex_userdata=userdata_worker,
ex_security_groups=["worker"]) ex_security_groups=["worker"])
print('Starting new app-worker-2 instance and wait until it is running...') print('Starting new app-worker-2 instance and wait until it is running...')
instance_worker_2 = conn.create_node(name='app-worker-2', instance_worker_2 = conn.create_node(location=az[1],
name='app-worker-2',
image=image, size=flavor, image=image, size=flavor,
ex_keyname=keypair_name, ex_keyname=keypair_name,
ex_userdata=userdata_worker, ex_userdata=userdata_worker,
...@@ -367,6 +399,7 @@ def main(): ...@@ -367,6 +399,7 @@ def main():
print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers())) print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers()))
for loadbalancer in elb_conn.list_balancers(): for loadbalancer in elb_conn.list_balancers():
if loadbalancer.name == "lb1": if loadbalancer.name == "lb1":
print("Deleting Load Balancer: " + str(loadbalancer))
elb_conn.destroy_balancer(loadbalancer) elb_conn.destroy_balancer(loadbalancer)
# get suffix (a, b, c, ...) from all availability zones, available in the selected region # get suffix (a, b, c, ...) from all availability zones, available in the selected region
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment