Skip to content
Snippets Groups Projects
Commit e18ab4dc authored by Sebastian Rieger's avatar Sebastian Rieger
Browse files

updated demo4 to support AWS Academy lab setup

parent afd3bef5
Branches
No related tags found
No related merge requests found
...@@ -10,20 +10,15 @@ from libcloud.compute.types import Provider, NodeState ...@@ -10,20 +10,15 @@ from libcloud.compute.types import Provider, NodeState
home = expanduser("~") home = expanduser("~")
# reqs:
# services: EC2 (nova, glance, neutron)
# resources: 2 instances (m1.small), 2 elastic ips (1 keypair, 2 security groups)
# default region # default region
# region_name = 'eu-central-1' # region_name = 'eu-central-1'
# region_name = 'ap-south-1' # region_name = 'ap-south-1'
# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com # AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE
region_name = 'us-east-1' region_name = 'us-east-1'
# keypairs are kept and not deleted, they do not cost anything anyway
def main(): def main():
########################################################################### ###########################################################################
......
...@@ -7,9 +7,11 @@ from libcloud.compute.types import Provider ...@@ -7,9 +7,11 @@ from libcloud.compute.types import Provider
home = expanduser("~") home = expanduser("~")
# reqs: # requirements:
# services: EC2 (nova, glance, neutron) # services: EC2
# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups) # resources: 2 instances (1 keypair, 2 security groups)
# optionally also elastic ip (comparable to floating ip) offering a persistent public
# IP, but elastic ips are expensive - make sure to delete them after you used them
# The image to look for and use for the started instance # The image to look for and use for the started instance
ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128' ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
...@@ -33,7 +35,7 @@ flavor_name = 't2.nano' ...@@ -33,7 +35,7 @@ flavor_name = 't2.nano'
region_name = 'us-east-1' region_name = 'us-east-1'
# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout # starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout
timeout = 300 timeout = 600
def main(): def main():
......
import configparser
from os.path import expanduser
# import getpass # import getpass
# import os # import os
# import libcloud.security # import libcloud.security
...@@ -12,33 +15,17 @@ from libcloud.loadbalancer.base import Member, Algorithm ...@@ -12,33 +15,17 @@ from libcloud.loadbalancer.base import Member, Algorithm
from libcloud.loadbalancer.types import Provider as loadbalancer_Provider from libcloud.loadbalancer.types import Provider as loadbalancer_Provider
from libcloud.loadbalancer.providers import get_driver as loadbalancer_get_driver from libcloud.loadbalancer.providers import get_driver as loadbalancer_get_driver
# reqs: home = expanduser("~")
# services: EC2 (nova, glance, neutron)
# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups)
# The image to look for and use for the started instance
# ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200408'
ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04 20200408 in us-west-1
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key
#
# id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
keypair_name = 'srieger-pub'
pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 't2.nano'
# default region # default region
# region_name = 'eu-central-1' # region_name = 'eu-central-1'
# region_name = 'ap-south-1' # region_name = 'ap-south-1'
# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com # AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE
region_name = 'us-east-1' region_name = 'us-east-1'
# keypairs are kept and not deleted, they do not cost anything anyway
def main(): def main():
########################################################################### ###########################################################################
...@@ -47,17 +34,18 @@ def main(): ...@@ -47,17 +34,18 @@ def main():
# #
########################################################################### ###########################################################################
# see AWS Educate classroom, Account Details # see AWS Academy Lab for Account Details
# read credentials from file
config = configparser.ConfigParser()
config.read_file(open(home + '/.aws/credentials'))
aws_access_key_id = config['default']['aws_access_key_id']
aws_secret_access_key = config['default']['aws_secret_access_key']
aws_session_token = config['default']['aws_session_token']
# aws_access_key_id = "ASIAX..."
# aws_secret_access_key = "eGwE12j..."
# aws_session_token = "FwoGZXIvYXdzEK///////////wEaDE..."
# access_id = getpass.win_getpass("Enter your access_id:")
# secret_key = getpass.win_getpass("Enter your secret_key:")
# session_token = getpass.win_getpass("Enter your session_token:")
# access_id = "ASIAU..."
# secret_key = "7lafW..."
# session_token = "IQoJb3JpZ...EMb//..."
access_id = "ASIA5ML7..."
secret_key = "76lAjn..."
session_token = "IQoJb3JpZ2luX2VjEBc..."
########################################################################### ###########################################################################
# #
...@@ -66,9 +54,9 @@ def main(): ...@@ -66,9 +54,9 @@ def main():
########################################################################### ###########################################################################
elb_provider = loadbalancer_get_driver(loadbalancer_Provider.ELB) elb_provider = loadbalancer_get_driver(loadbalancer_Provider.ELB)
elb_conn = elb_provider(access_id, elb_conn = elb_provider(aws_access_key_id,
secret_key, aws_secret_access_key,
token=session_token, token=aws_session_token,
region=region_name) region=region_name)
print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers())) print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers()))
...@@ -83,9 +71,9 @@ def main(): ...@@ -83,9 +71,9 @@ def main():
########################################################################### ###########################################################################
provider = compute_get_driver(compute_Provider.EC2) provider = compute_get_driver(compute_Provider.EC2)
conn = provider(key=access_id, conn = provider(key=aws_access_key_id,
secret=secret_key, secret=aws_secret_access_key,
token=session_token, token=aws_session_token,
region=region_name) region=region_name)
########################################################################### ###########################################################################
...@@ -102,7 +90,7 @@ def main(): ...@@ -102,7 +90,7 @@ def main():
print('Destroying Instance: %s' % instance.name) print('Destroying Instance: %s' % instance.name)
conn.destroy_node(instance) conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups # wait until all nodes are destroyed to be able to remove dependent security groups
nodes_still_running = True nodes_still_running = True
while nodes_still_running: while nodes_still_running:
nodes_still_running = False nodes_still_running = False
...@@ -115,19 +103,30 @@ def main(): ...@@ -115,19 +103,30 @@ def main():
nodes_still_running = True nodes_still_running = True
print('There are still instances running, waiting for them to be destroyed...') print('There are still instances running, waiting for them to be destroyed...')
# delete security groups, respecting dependencies (hence deleting 'control' and 'services' first) # delete security groups
for group in conn.ex_list_security_groups(): for group in conn.ex_list_security_groups():
if group in ['control', 'services']: # services depends on worker and api, so delete services first...
if group in ['services']:
print('Deleting security group: %s' % group) print('Deleting security group: %s' % group)
conn.ex_delete_security_group(group) conn.ex_delete_security_group(group)
# now we can delete security groups 'api' and 'worker', as 'control' and 'api' depended on them, otherwise AWS will
# throw DependencyViolation: resource has a dependent object
for group in conn.ex_list_security_groups(): for group in conn.ex_list_security_groups():
if group in ['api', 'worker']: # control depends on worker, so delete control before worker...
if group in ['control']:
print('Deleting security group: %s' % group) print('Deleting security group: %s' % group)
conn.ex_delete_security_group(group) conn.ex_delete_security_group(group)
for group in conn.ex_list_security_groups():
if group in ['worker', 'api']:
print('Deleting security group: %s' % group)
conn.ex_delete_security_group(group)
# release elastic ips
for elastic_ip in conn.ex_describe_all_addresses():
if elastic_ip.instance_id is None:
print('Releasing unused elastic ip %s' % elastic_ip)
conn.ex_release_address(elastic_ip, domain=elastic_ip.domain)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
# import getpass import configparser
# import os from os.path import expanduser
# import libcloud.security # import libcloud.security
import time import time
...@@ -12,13 +13,16 @@ from libcloud.loadbalancer.base import Member, Algorithm ...@@ -12,13 +13,16 @@ from libcloud.loadbalancer.base import Member, Algorithm
from libcloud.loadbalancer.types import Provider as loadbalancer_Provider from libcloud.loadbalancer.types import Provider as loadbalancer_Provider
from libcloud.loadbalancer.providers import get_driver as loadbalancer_get_driver from libcloud.loadbalancer.providers import get_driver as loadbalancer_get_driver
# reqs: home = expanduser("~")
# services: EC2 (nova, glance, neutron)
# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups) # requirements:
# services: EC2, ELB
# resources: 2 instances, (1 keypair, 2 security groups),
# 1 (Classic) Elastic Load Balancer, expensive! delete it after you used it!
# The image to look for and use for the started instance # The image to look for and use for the started instance
# ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200408' ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04 20200408 in us-west-1 # TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3...
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
# #
...@@ -26,7 +30,7 @@ ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04 ...@@ -26,7 +30,7 @@ ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
keypair_name = 'srieger-pub' keypair_name = 'srieger-pub'
pub_key_file = '~/.ssh/id_rsa.pub' pub_key_file = home + '/.ssh/id_rsa.pub'
flavor_name = 't2.nano' flavor_name = 't2.nano'
...@@ -34,11 +38,12 @@ flavor_name = 't2.nano' ...@@ -34,11 +38,12 @@ flavor_name = 't2.nano'
# region_name = 'eu-central-1' # region_name = 'eu-central-1'
# region_name = 'ap-south-1' # region_name = 'ap-south-1'
# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com # AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE
region_name = 'us-east-1' region_name = 'us-east-1'
# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout
timeout = 600
def main(): def main():
########################################################################### ###########################################################################
...@@ -47,17 +52,19 @@ def main(): ...@@ -47,17 +52,19 @@ def main():
# #
########################################################################### ###########################################################################
# see AWS Educate classroom, Account Details # see AWS Academy Lab for Account Details
# read credentials from file
config = configparser.ConfigParser()
config.read_file(open(home + '/.aws/credentials'))
aws_access_key_id = config['default']['aws_access_key_id']
aws_secret_access_key = config['default']['aws_secret_access_key']
aws_session_token = config['default']['aws_session_token']
# hard coded AWS credentials using vars
# aws_access_key_id = "ASIAX..."
# aws_secret_access_key = "WLxxXK+..."
# aws_session_token = "FwoGZXIvYXdzEMb//////////wEaDE5rX.......0SleZ+L75I9iEri9LA4hovWul8HvexhCBK8.......................Ae/T+VkUbcQRtJEDwg+gYCABuk0JlSj5Wk7YA65r3BSNJXZFpkhbek6VBjvE/cEt5fKZEhENcdFxjAcAJLd6bOWi/oGXU5e3PX3mcXgm0oJpz6h3wqD1LvSDtw5GDwn0BHiF1Mu.......................cm/VukK5F"
# access_id = getpass.win_getpass("Enter your access_id:")
# secret_key = getpass.win_getpass("Enter your secret_key:")
# session_token = getpass.win_getpass("Enter your session_token:")
# access_id = "ASIAU..."
# secret_key = "7lafW..."
# session_token = "IQoJb3JpZ...EMb//..."
access_id = "ASIA..."
secret_key = "76lAj..."
session_token = "IQoJ..."
########################################################################### ###########################################################################
# #
...@@ -66,9 +73,9 @@ def main(): ...@@ -66,9 +73,9 @@ def main():
########################################################################### ###########################################################################
provider = compute_get_driver(compute_Provider.EC2) provider = compute_get_driver(compute_Provider.EC2)
conn = provider(key=access_id, conn = provider(key=aws_access_key_id,
secret=secret_key, secret=aws_secret_access_key,
token=session_token, token=aws_session_token,
region=region_name) region=region_name)
########################################################################### ###########################################################################
...@@ -77,15 +84,32 @@ def main(): ...@@ -77,15 +84,32 @@ def main():
# #
########################################################################### ###########################################################################
print("Search for AMI...")
image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0]
print("Using image: %s" % image)
# print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of " # print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of "
# "supported operating systems and versions)... please be patient...") # "supported operating systems and versions)... please be patient...")
# images = conn.list_images() # image = ''
# for img in images:
# # if img.name == ubuntu_image_name:
# if img.extra['owner_alias'] == 'amazon':
# print(img)
# if img.id == ubuntu_image_name:
# image = img
# fetch/select the image referenced with ubuntu_image_name above # fetch/select the image referenced with ubuntu_image_name above
# image = [i for i in images if i.name == ubuntu_image_name][0] # image = [i for i in images if i.name == ubuntu_image_name][0]
# print(image) # print(image)
# selecting the image based on defined AMI id # select image directly to save time, as retrieving the image list takes several minutes now,
image = NodeImage(id=ubuntu_image_id, name=None, driver=conn) # need to change ami id here if updated or for other regions, id is working for course in
# summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard,
# to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the
# desired region
# image = NodeImage(id="ami-0e472ba40eb589f49",
# name=ubuntu_image_name,
# driver="hvm")
flavors = conn.list_sizes() flavors = conn.list_sizes()
flavor = [s for s in flavors if s.id == flavor_name][0] flavor = [s for s in flavors if s.id == flavor_name][0]
...@@ -132,7 +156,7 @@ def main(): ...@@ -132,7 +156,7 @@ def main():
print('Destroying Instance: %s' % instance.name) print('Destroying Instance: %s' % instance.name)
conn.destroy_node(instance) conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups # wait until all nodes are destroyed to be able to remove dependent security groups
nodes_still_running = True nodes_still_running = True
while nodes_still_running: while nodes_still_running:
nodes_still_running = False nodes_still_running = False
...@@ -234,7 +258,7 @@ def main(): ...@@ -234,7 +258,7 @@ def main():
# Thanks to Stefan Friedmann for finding this fix ;) # Thanks to Stefan Friedmann for finding this fix ;)
userdata_service = '''#!/usr/bin/env bash userdata_service = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
-i database -i messaging -i database -i messaging
rabbitmqctl add_user faafo guest rabbitmqctl add_user faafo guest
rabbitmqctl set_user_tags faafo administrator rabbitmqctl set_user_tags faafo administrator
...@@ -258,7 +282,7 @@ def main(): ...@@ -258,7 +282,7 @@ def main():
########################################################################### ###########################################################################
userdata_api = '''#!/usr/bin/env bash userdata_api = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
-i faafo -r api -m 'amqp://faafo:guest@%(services_ip)s:5672/' \ -i faafo -r api -m 'amqp://faafo:guest@%(services_ip)s:5672/' \
-d 'mysql+pymysql://faafo:password@%(services_ip)s:3306/faafo' -d 'mysql+pymysql://faafo:password@%(services_ip)s:3306/faafo'
''' % {'services_ip': services_ip} ''' % {'services_ip': services_ip}
...@@ -291,7 +315,7 @@ def main(): ...@@ -291,7 +315,7 @@ def main():
########################################################################### ###########################################################################
userdata_worker = '''#!/usr/bin/env bash userdata_worker = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
-i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/' -i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/'
''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip} ''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip}
...@@ -335,9 +359,9 @@ def main(): ...@@ -335,9 +359,9 @@ def main():
########################################################################### ###########################################################################
elb_provider = loadbalancer_get_driver(loadbalancer_Provider.ELB) elb_provider = loadbalancer_get_driver(loadbalancer_Provider.ELB)
elb_conn = elb_provider(access_id, elb_conn = elb_provider(aws_access_key_id,
secret_key, aws_secret_access_key,
token=session_token, token=aws_session_token,
region=region_name) region=region_name)
print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers())) print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers()))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment