diff --git a/scripts/perf-test/codebuild.yaml b/scripts/perf-test/codebuild.yaml new file mode 100644 index 00000000000..45663349b61 --- /dev/null +++ b/scripts/perf-test/codebuild.yaml @@ -0,0 +1,127 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + S3Bucket: + Type: String + + PerfTestId: + Type: String + + RepoType: + Type: String + + Repository: + Type: String + +Resources: + CodeBuildRole: + Type: "AWS::IAM::Role" + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: "sts:AssumeRole" + RoleName: !Sub "CR-${PerfTestId}" + Policies: + - PolicyName: !Sub "CP-${PerfTestId}" + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - "s3:PutObject" + Effect: Allow + Resource: !Join ["/", [!Sub "arn:aws:s3:::${S3Bucket}", "*"]] + - Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + Effect: Allow + Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/*' + + ReleaseBuild: + Type: "AWS::CodeBuild::Project" + Properties: + Artifacts: + Type: S3 + Location: !Ref S3Bucket + Path: !Ref PerfTestId + Name: release + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: aws/codebuild/ubuntu-base:14.04 + Type: LINUX_CONTAINER + Name: !Sub "perf-test-release-build-${PerfTestId}" + ServiceRole: !Ref CodeBuildRole + Source: + BuildSpec: !Sub | + version: 0.2 + phases: + install: + commands: + - apt-get update -y + - apt-get install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test + - apt-get update -y + - apt-get install -y libwww-perl g++-5 flex bison git + - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 1 + - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 1 + build: + commands: + - echo ${Repository} > COMMIT_INFO + - git rev-parse --short HEAD >> COMMIT_INFO + - git log HEAD^..HEAD >> COMMIT_INFO + - make -C src minisat2-download glucose-download + - make -C src -j8 + artifacts: + files: + - src/cbmc/cbmc + - COMMIT_INFO + discard-paths: yes + Type: !Ref RepoType + Location: !Ref Repository + + ProfilingBuild: + Type: "AWS::CodeBuild::Project" + Properties: + Artifacts: + Type: S3 + Location: !Ref S3Bucket + Path: !Ref PerfTestId + Name: profiling + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: aws/codebuild/ubuntu-base:14.04 + Type: LINUX_CONTAINER + Name: !Sub "perf-test-profiling-build-${PerfTestId}" + ServiceRole: !Ref CodeBuildRole + Source: + BuildSpec: !Sub | + version: 0.2 + phases: + install: + commands: + - apt-get update -y + - apt-get install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test + - apt-get update -y + - apt-get install -y libwww-perl g++-5 flex bison git + - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 1 + - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 1 + build: + commands: + - echo ${Repository} > COMMIT_INFO + - git rev-parse --short HEAD >> COMMIT_INFO + - git log HEAD^..HEAD >> COMMIT_INFO + - make -C src minisat2-download glucose-download + - make -C src -j8 CXXFLAGS="-O2 -pg -g -finline-limit=4" LINKFLAGS="-pg" + artifacts: + files: + - src/cbmc/cbmc + - COMMIT_INFO + discard-paths: yes + Type: !Ref RepoType + Location: !Ref Repository diff --git a/scripts/perf-test/ebs.yaml b/scripts/perf-test/ebs.yaml new file mode 100644 index 00000000000..609554c8f39 --- /dev/null +++ b/scripts/perf-test/ebs.yaml @@ -0,0 +1,53 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + Ami: + Type: String + + AvailabilityZone: + Type: String + +Resources: + EC2Instance: + Type: "AWS::EC2::Instance" + Properties: + InstanceType: t2.micro + ImageId: !Ref Ami + AvailabilityZone: !Ref AvailabilityZone + Volumes: + - Device: "/dev/sdf" + VolumeId: !Ref BaseVolume + UserData: !Base64 | + #!/bin/bash + set -e + # wait to make sure volume is available + sleep 10 + mkfs.ext4 /dev/xvdf + mount /dev/xvdf /mnt + apt-get -y update + apt-get install git + cd /mnt + git clone --depth 1 --branch svcomp17 \ + https://github.com/sosy-lab/sv-benchmarks.git + git clone --depth 1 \ + https://github.com/sosy-lab/benchexec.git + git clone --depth 1 --branch trunk \ + https://github.com/sosy-lab/cpachecker.git + git clone --depth 1 \ + https://github.com/diffblue/cprover-sv-comp.git + halt + + BaseVolume: + Type: "AWS::EC2::Volume" + DeletionPolicy: Snapshot + Properties: + AvailabilityZone: !Ref AvailabilityZone + Size: 8 + Tags: + - Key: Name + Value: perf-test-base + +Outputs: + InstanceId: + Value: !Ref EC2Instance diff --git a/scripts/perf-test/ec2.yaml b/scripts/perf-test/ec2.yaml new file mode 100644 index 00000000000..5933777f6e2 --- /dev/null +++ b/scripts/perf-test/ec2.yaml @@ -0,0 +1,394 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + InstanceType: + Type: String + + Ami: + Type: String + + SnapshotId: + Type: String + + AvailabilityZone: + Type: String + + S3Bucket: + Type: String + + PerfTestId: + Type: String + + SnsTopic: + Type: String + + SqsArn: + Type: String + + SqsUrl: + Type: String + + MaxPrice: + Type: String + + FleetSize: + Type: String + + SSHKeyName: + Type: String + +Conditions: + UseSpot: !Not [!Equals [!Ref MaxPrice, ""]] + + UseKey: !Not [!Equals [!Ref SSHKeyName, ""]] + +Resources: + EC2Role: + Type: "AWS::IAM::Role" + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: "sts:AssumeRole" + RoleName: !Sub "ER-${PerfTestId}" + Policies: + - PolicyName: !Sub "EP-${PerfTestId}" + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - "s3:PutObject" + - "s3:GetObject" + Effect: Allow + Resource: !Join ["/", [!Sub "arn:aws:s3:::${S3Bucket}", "*"]] + - Action: + - "sns:Publish" + Effect: Allow + Resource: !Ref SnsTopic + - Action: + - "sqs:DeleteMessage" + - "sqs:DeleteQueue" + - "sqs:GetQueueAttributes" + - "sqs:ReceiveMessage" + Effect: Allow + Resource: !Ref SqsArn + - Action: + - "sqs:DeleteMessage" + - "sqs:DeleteQueue" + - "sqs:GetQueueAttributes" + - "sqs:ReceiveMessage" + - "sqs:SendMessage" + Effect: Allow + Resource: !Sub "${SqsArn}-run" + - Action: + - "cloudformation:DeleteStack" + Effect: Allow + Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/perf-test-*/*" + - Action: + - "autoscaling:DeleteAutoScalingGroup" + - "autoscaling:DeleteLaunchConfiguration" + - "autoscaling:DescribeAutoScalingGroups" + - "autoscaling:DescribeScalingActivities" + - "autoscaling:UpdateAutoScalingGroup" + - "ec2:DeleteSecurityGroup" + - "iam:DeleteInstanceProfile" + - "iam:DeleteRole" + - "iam:DeleteRolePolicy" + - "iam:RemoveRoleFromInstanceProfile" + Effect: Allow + Resource: "*" + + EC2InstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Roles: + - !Ref EC2Role + + SecurityGroupInSSHWorld: + Type: "AWS::EC2::SecurityGroup" + DependsOn: EC2Role + Properties: + GroupDescription: SSH access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: '0.0.0.0/0' + + LaunchConfiguration: + Type: "AWS::AutoScaling::LaunchConfiguration" + Properties: + BlockDeviceMappings: + - DeviceName: "/dev/sdf" + Ebs: + DeleteOnTermination: True + SnapshotId: !Ref SnapshotId + VolumeSize: 64 + IamInstanceProfile: !Ref EC2InstanceProfile + ImageId: !Ref Ami + InstanceType: !Ref InstanceType + KeyName: + !If [UseKey, !Ref SSHKeyName, !Ref "AWS::NoValue"] + SecurityGroups: + - !Ref SecurityGroupInSSHWorld + SpotPrice: + !If [UseSpot, !Ref MaxPrice, !Ref "AWS::NoValue"] + UserData: + Fn::Base64: !Sub | + #!/bin/bash + set -x -e + + # wait to make sure volume is available + sleep 10 + e2fsck -f -y /dev/xvdf + resize2fs /dev/xvdf + mount /dev/xvdf /mnt + + # install packages + apt-get -y update + apt-get install -y git time wget binutils awscli make jq + apt-get install -y gcc libc6-dev-i386 + + # cgroup set up for benchexec + chmod o+wt '/sys/fs/cgroup/cpuset/' + chmod o+wt '/sys/fs/cgroup/cpu,cpuacct/user.slice' + chmod o+wt '/sys/fs/cgroup/memory/user.slice' + chmod o+wt '/sys/fs/cgroup/freezer/' + + # AWS Sig-v4 access + aws configure set s3.signature_version s3v4 + + # send instance-terminated message + # http://rogueleaderr.com/post/48795010760/how-to-notifyemail-yourself-when-an-ec2-instance/amp + cat >/etc/init.d/ec2-terminate <<"EOF" + #!/bin/bash + ### BEGIN INIT INFO + # Provides: ec2-terminate + # Required-Start: $network $syslog + # Required-Stop: + # Default-Start: + # Default-Stop: 0 1 6 + # Short-Description: ec2-terminate + # Description: send termination email + ### END INIT INFO + # + + case "$1" in + start|status) + exit 0 + ;; + stop) + # run the below + ;; + *) + exit 1 + ;; + esac + + ut=$(cat /proc/uptime | cut -f1 -d" ") + aws --region us-east-1 sns publish \ + --topic-arn ${SnsTopic} \ + --message "instance terminating after $ut s at ${MaxPrice} USD/h" + sleep 3 # make sure the message has time to send + aws s3 cp /var/log/cloud-init-output.log \ + s3://${S3Bucket}/${PerfTestId}/$HOSTNAME.cloud-init-output.log + + exit 0 + EOF + chmod a+x /etc/init.d/ec2-terminate + update-rc.d ec2-terminate defaults + systemctl start ec2-terminate + + # prepare for tool packaging + cd /mnt + cd cprover-sv-comp + mkdir -p src/cbmc/ + touch LICENSE + cd .. + mkdir -p run + cd run + wget -O cbmc.xml https://raw.githubusercontent.com/sosy-lab/sv-comp/master/benchmark-defs/cbmc.xml + sed -i 's/witness.graphml/${!logfile_path_abs}${!inputfile_name}-witness.graphml/' cbmc.xml + cd .. + mkdir -p tmp + export TMPDIR=/mnt/tmp + + # reduce the likelihood of multiple hosts processing the + # same message (in addition to SQS's message hiding) + sleep $(expr $RANDOM % 30) + retry=1 + + while true + do + sqs=$(aws --region ${AWS::Region} sqs receive-message \ + --queue-url ${SqsUrl} | \ + jq -r '.Messages[0].Body,.Messages[0].ReceiptHandle') + + if [ -z "$sqs" ] + then + # no un-read messages in the input queue; let's look + # at -run + n_msgs=$(aws --region ${AWS::Region} sqs \ + get-queue-attributes \ + --queue-url ${SqsUrl}-run \ + --attribute-names \ + ApproximateNumberOfMessages | \ + jq -r '.Attributes.ApproximateNumberOfMessages') + + if [ $retry -eq 1 ] + then + retry=0 + sleep 30 + continue + elif [ -n "$n_msgs" ] && [ "$n_msgs" = "0" ] + then + # shut down the infrastructure + aws --region us-east-1 sns publish \ + --topic-arn ${SnsTopic} \ + --message "Trying to delete stacks in ${AWS::Region}" + aws --region ${AWS::Region} cloudformation \ + delete-stack --stack-name \ + perf-test-sqs-${PerfTestId} + aws --region ${AWS::Region} cloudformation \ + delete-stack --stack-name \ + perf-test-exec-${PerfTestId} + halt + fi + + # the queue is gone, or other host will be turning + # off the lights + halt + fi + + retry=1 + bm=$(echo $sqs | cut -f1 -d" ") + cfg=$(echo $bm | cut -f1 -d"-") + t=$(echo $bm | cut -f2- -d"-") + msg=$(echo $sqs | cut -f2- -d" ") + + # mark $bm in-progress + aws --region ${AWS::Region} sqs send-message \ + --queue-url ${SqsUrl}-run \ + --message-body $bm-$(hostname) + + # there is no guarantee of cross-queue action ordering + # sleep for a bit to reduce the likelihood of missing + # in-progress messages while the input queue is empty + sleep 3 + + # remove it from the input queue + aws --region ${AWS::Region} sqs delete-message \ + --queue-url ${SqsUrl} \ + --receipt-handle $msg + + cd /mnt/cprover-sv-comp + rm -f src/cbmc/cbmc + aws s3 cp s3://${S3Bucket}/${PerfTestId}/$cfg/cbmc \ + src/cbmc/cbmc + chmod a+x src/cbmc/cbmc + make CBMC=. YEAR=N CBMC-sv-comp-N.tar.gz + cd ../run + tar xzf ../cprover-sv-comp/CBMC-sv-comp-N.tar.gz + rm ../cprover-sv-comp/CBMC-sv-comp-N.tar.gz + + date + echo "Task: $t" + + # compute the number of possible executors + max_par=$(cat /proc/cpuinfo | grep ^processor | wc -l) + mem=$(free -g | grep ^Mem | awk '{print $2}') + if [ $cfg != "profiling" ] + then + mem=$(expr $mem / 15) + else + mem=$(expr $mem / 7) + fi + if [ $mem -lt $max_par ] + then + max_par=$mem + fi + + if [ $cfg != "profiling" ] + then + ../benchexec/bin/benchexec cbmc.xml --no-container \ + --task $t -T 900s -M 15GB -o logs-$t/ \ + -N $max_par -c 1 + if [ -d logs-$t/cbmc.*.logfiles ] + then + cd logs-$t + tar czf witnesses.tar.gz cbmc.*.logfiles + rm -rf cbmc.*.logfiles + cd .. + fi + if [ -f logs-$t/*.xml.bz2 ] + then + start_date="$(echo ${PerfTestId} | cut -f1-3 -d-) $(echo ${PerfTestId} | cut -f4-6 -d- | sed 's/-/:/g')" + cd logs-$t + bunzip2 *.xml.bz2 + perl -p -i -e \ + "s/^(/dev/null 2>&1 + then + gprof --sum ./cbmc-binary *.gmon.out.* + gprof ./cbmc-binary gmon.sum > sum.profile-$t + rm -f gmon.sum gmon.out *.gmon.out.* + aws s3 cp sum.profile-$t \ + s3://${S3Bucket}/${PerfTestId}/$cfg/sum.profile-$t + fi + fi + rm -rf logs-$t sum.profile-$t + date + + # clear out the in-progress message + while true + do + sqs=$(aws --region ${AWS::Region} sqs \ + receive-message \ + --queue-url ${SqsUrl}-run \ + --visibility-timeout 10 | \ + jq -r '.Messages[0].Body,.Messages[0].ReceiptHandle') + bm2=$(echo $sqs | cut -f1 -d" ") + msg2=$(echo $sqs | cut -f2- -d" ") + + if [ "$bm2" = "$bm-$(hostname)" ] + then + aws --region ${AWS::Region} sqs delete-message \ + --queue-url ${SqsUrl}-run \ + --receipt-handle $msg2 + break + fi + done + done + + AutoScalingGroup: + Type: "AWS::AutoScaling::AutoScalingGroup" + Properties: + AvailabilityZones: + - !Ref AvailabilityZone + DesiredCapacity: !Ref FleetSize + LaunchConfigurationName: !Ref LaunchConfiguration + MaxSize: !Ref FleetSize + MinSize: 1 + +Outputs: + ASGId: + Value: !Ref AutoScalingGroup diff --git a/scripts/perf-test/perf_test.py b/scripts/perf-test/perf_test.py new file mode 100755 index 00000000000..0f4de36dc40 --- /dev/null +++ b/scripts/perf-test/perf_test.py @@ -0,0 +1,669 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import argparse +import boto3 +import concurrent.futures +import contextlib +import datetime +import json +import logging +import os +import re +import shutil +import sys +import tempfile +import time +import urllib + + +@contextlib.contextmanager +def make_temp_directory(): + """ + create a temporary directory and remove it once the statement completes + """ + temp_dir = tempfile.mkdtemp() + try: + yield temp_dir + finally: + shutil.rmtree(temp_dir) + + +def same_dir(filename): + d = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(d, filename) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-r', '--repository', type=str, required=True, + help='CodeCommit or GitHub repository containing ' + + 'code to evaluate') + parser.add_argument('-c', '--commit-id', type=str, required=True, + help='git revision to evaluate') + parser.add_argument('-e', '--email', type=str, required=True, + help='Email address to notify about results') + parser.add_argument('-t', '--instance-type', type=str, + default='r4.16xlarge', + help='Amazon EC2 instance type to use ' + + '(default: r4.16xlarge)') + parser.add_argument('-m', '--mode', + choices=['spot', 'on-demand', 'batch'], + default='spot', + help='Choose fleet to run benchmarks on ' + + '(default: Amazon EC2 Spot fleet)') + parser.add_argument('-R', '--region', type=str, + help='Set fixed region instead of cheapest fleet') + parser.add_argument('-j', '--parallel', type=int, default=4, + help='Fleet size of concurrently running hosts ' + + '(default: 4)') + parser.add_argument('-k', '--ssh-key-name', type=str, default='', + help='EC2 key name for SSH access to fleet') + parser.add_argument('-K', '--ssh-key', type=str, + help='SSH public key file for access to fleet ' + + '(requires -K/--ssh-key-name)') + parser.add_argument('-T', '--tasks', type=str, + default='quick', + help='Subset of tasks to run (quick, full; ' + + 'default: quick; or name of SV-COMP task)') + + args = parser.parse_args() + assert(args.repository.startswith('https://github.com/') or + args.repository.startswith('https://git-codecommit.')) + assert(not args.ssh_key or args.ssh_key_name) + if args.ssh_key: + assert(os.path.isfile(args.ssh_key)) + + return args + + +def prepare_s3(session, bucket_name, artifact_uploaded_arn): + # create a bucket for storing artifacts + logger = logging.getLogger('perf_test') + s3 = session.resource('s3', region_name='us-east-1') + buckets = list(s3.buckets.all()) + + for b in buckets: + if b.name == bucket_name: + logger.info('us-east-1: S3 bucket {} exists'.format(bucket_name)) + return + + cfn = session.resource('cloudformation', region_name='us-east-1') + with open(same_dir('s3.yaml')) as f: + CFN_s3 = f.read() + cfn.create_stack( + StackName='perf-test-s3', + TemplateBody=CFN_s3, + Parameters=[ + { + 'ParameterKey': 'SnsTopicArn', + 'ParameterValue': artifact_uploaded_arn + }, + { + 'ParameterKey': 'S3BucketName', + 'ParameterValue': bucket_name + } + ]) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName='perf-test-s3', WaiterConfig={'Delay': 10}) + logger.info('us-east-1: S3 bucket {} set up'.format(bucket_name)) + + +def prepare_sns_s3(session, email, bucket_name): + # create instance_terminated topic + # create artifact_uploaded topic + logger = logging.getLogger('perf_test') + sns = session.resource('sns', region_name='us-east-1') + topics = list(sns.topics.all()) + instance_terminated_arn = None + artifact_uploaded_arn = None + + for t in topics: + if t.attributes['DisplayName'] == 'instance_terminated': + instance_terminated_arn = t.arn + logger.info('us-east-1: SNS topic instance_terminated exists') + if int(t.attributes['SubscriptionsPending']) > 0: + logger.warning('us-east-1: SNS topic instance_terminated ' + + 'has pending subscription confirmations') + elif t.attributes['DisplayName'] == 'artifact_uploaded': + artifact_uploaded_arn = t.arn + logger.info('us-east-1: SNS topic artifact_uploaded exists') + if int(t.attributes['SubscriptionsPending']) > 0: + logger.warning('us-east-1: SNS topic artifact_uploaded ' + + 'has pending subscription confirmations') + + cfn = session.resource('cloudformation', region_name='us-east-1') + + with open(same_dir('sns.yaml')) as f: + CFN_sns = f.read() + + if not instance_terminated_arn: + cfn.create_stack( + StackName='perf-test-sns-instance-term', + TemplateBody=CFN_sns, + Parameters=[ + { + 'ParameterKey': 'NotificationAddress', + 'ParameterValue': email + }, + { + 'ParameterKey': 'SnsTopicName', + 'ParameterValue': 'instance_terminated' + } + ]) + + if not artifact_uploaded_arn: + cfn.create_stack( + StackName='perf-test-sns-artifact-uploaded', + TemplateBody=CFN_sns, + Parameters=[ + { + 'ParameterKey': 'NotificationAddress', + 'ParameterValue': email + }, + { + 'ParameterKey': 'SnsTopicName', + 'ParameterValue': 'artifact_uploaded' + } + ]) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + if not instance_terminated_arn: + waiter.wait( + StackName='perf-test-sns-instance-term', + WaiterConfig={'Delay': 10}) + stack = cfn.Stack('perf-test-sns-instance-term') + instance_terminated_arn = stack.outputs[0]['OutputValue'] + logger.info('us-east-1: SNS topic instance_terminated set up') + if not artifact_uploaded_arn: + waiter.wait( + StackName='perf-test-sns-artifact-uploaded', + WaiterConfig={'Delay': 10}) + stack = cfn.Stack('perf-test-sns-artifact-uploaded') + artifact_uploaded_arn = stack.outputs[0]['OutputValue'] + logger.info('us-east-1: SNS topic artifact_uploaded set up') + + prepare_s3(session, bucket_name, artifact_uploaded_arn) + + return instance_terminated_arn + + +def select_region(session, mode, region, instance_type): + # find the region and az with the lowest spot price for the chosen instance + # type + # based on https://gist.github.com/pahud/fbbc1fd80fac4544fd0a3a480602404e + logger = logging.getLogger('perf_test') + + if not region: + ec2 = session.client('ec2') + regions = [r['RegionName'] for r in ec2.describe_regions()['Regions']] + else: + regions = [region] + + min_region = None + min_az = None + min_price = None + + if mode == 'on-demand': + logger.info('global: Fetching on-demand prices for ' + instance_type) + with make_temp_directory() as tmp_dir: + for r in regions: + json_file = os.path.join(tmp_dir, 'index.json') + urllib.request.urlretrieve( + 'https://pricing.us-east-1.amazonaws.com/offers/' + + 'v1.0/aws/AmazonEC2/current/' + r + '/index.json', + json_file) + with open(json_file) as jf: + json_result = json.load(jf) + key = None + for p in json_result['products']: + v = json_result['products'][p] + a = v['attributes'] + if ((v['productFamily'] == 'Compute Instance') and + (a['instanceType'] == instance_type) and + (a['tenancy'] == 'Shared') and + (a['operatingSystem'] == 'Linux')): + assert(not key) + key = p + for c in json_result['terms']['OnDemand'][key]: + v = json_result['terms']['OnDemand'][key][c] + for p in v['priceDimensions']: + price = v['priceDimensions'][p]['pricePerUnit']['USD'] + if min_region is None or float(price) < min_price: + min_region = r + ec2 = session.client('ec2', region_name=r) + azs = ec2.describe_availability_zones( + Filters=[{'Name': 'region-name', 'Values': [r]}]) + min_az = azs['AvailabilityZones'][0]['ZoneName'] + min_price = float(price) + + logger.info('global: Lowest on-demand price: {} ({}): {}'.format( + min_region, min_az, min_price)) + else: + logger.info('global: Fetching spot prices for ' + instance_type) + for r in regions: + ec2 = session.client('ec2', region_name=r) + res = ec2.describe_spot_price_history( + InstanceTypes=[instance_type], + ProductDescriptions=['Linux/UNIX'], + StartTime=datetime.datetime.now()) + history = res['SpotPriceHistory'] + for az in history: + if min_region is None or float(az['SpotPrice']) < min_price: + min_region = r + min_az = az['AvailabilityZone'] + min_price = float(az['SpotPrice']) + + logger.info('global: Lowest spot price: {} ({}): {}'.format( + min_region, min_az, min_price)) + + # http://aws-ubuntu.herokuapp.com/ + # 20170919 - Ubuntu 16.04 LTS (xenial) - hvm:ebs-ssd + AMI_ids = { + "Mappings": { + "RegionMap": { + "ap-northeast-1": {"64": "ami-8422ebe2"}, + "ap-northeast-2": {"64": "ami-0f6fb461"}, + "ap-south-1": {"64": "ami-08a5e367"}, + "ap-southeast-1": {"64": "ami-e6d3a585"}, + "ap-southeast-2": {"64": "ami-391ff95b"}, + "ca-central-1": {"64": "ami-e59c2581"}, + "eu-central-1": {"64": "ami-5a922335"}, + "eu-west-1": {"64": "ami-17d11e6e"}, + "eu-west-2": {"64": "ami-e1f2e185"}, + "sa-east-1": {"64": "ami-a3e39ecf"}, + "us-east-1": {"64": "ami-d651b8ac"}, + "us-east-2": {"64": "ami-9686a4f3"}, + "us-west-1": {"64": "ami-2d5c6d4d"}, + "us-west-2": {"64": "ami-ecc63a94"} + } + } + } + + ami = AMI_ids['Mappings']['RegionMap'][min_region]['64'] + + return (min_region, min_az, min_price, ami) + + +def prepare_ebs(session, region, az, ami): + # create an ebs volume that contains the benchmark sources + logger = logging.getLogger('perf_test') + ec2 = session.client('ec2', region_name=region) + snapshots = ec2.describe_snapshots( + OwnerIds=['self'], + Filters=[ + { + 'Name': 'tag:Name', + 'Values': ['perf-test-base'] + } + ]) + + if snapshots['Snapshots']: + logger.info(region + ': EBS snapshot exists') + else: + logger.info(region + ': EBS snapshot preparation required') + cfn = session.resource('cloudformation', region_name=region) + with open(same_dir('ebs.yaml')) as f: + CFN_ebs = f.read() + stack = cfn.create_stack( + StackName='perf-test-build-ebs', + TemplateBody=CFN_ebs, + Parameters=[ + { + 'ParameterKey': 'Ami', + 'ParameterValue': ami + }, + { + 'ParameterKey': 'AvailabilityZone', + 'ParameterValue': az + } + ]) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName='perf-test-build-ebs') + instance_id = stack.outputs[0]['OutputValue'] + logger.info(region + ': Waiting for EBS snapshot preparation on ' + + instance_id) + waiter = ec2.get_waiter('instance_stopped') + waiter.wait(InstanceIds=[instance_id]) + stack.delete() + waiter = cfn.meta.client.get_waiter('stack_delete_complete') + waiter.wait(StackName='perf-test-build-ebs') + logger.info(region + ': EBS snapshot prepared') + + snapshots = ec2.describe_snapshots( + OwnerIds=['self'], + Filters=[ + { + 'Name': 'tag:Name', + 'Values': ['perf-test-base'] + } + ]) + + return snapshots['Snapshots'][0]['SnapshotId'] + + +def build(session, repository, commit_id, bucket_name, perf_test_id): + # build the chosen commit in CodeBuild + logger = logging.getLogger('perf_test') + + if repository.startswith('https://github.com/'): + repo_type = 'GITHUB' + else: + repo_type = 'CODECOMMIT' + + cfn = session.resource('cloudformation', region_name='us-east-1') + stack_name = 'perf-test-codebuild-' + perf_test_id + with open(same_dir('codebuild.yaml')) as f: + CFN_codebuild = f.read() + stack = cfn.create_stack( + StackName=stack_name, + TemplateBody=CFN_codebuild, + Parameters=[ + { + 'ParameterKey': 'S3Bucket', + 'ParameterValue': bucket_name + }, + { + 'ParameterKey': 'PerfTestId', + 'ParameterValue': perf_test_id + }, + { + 'ParameterKey': 'RepoType', + 'ParameterValue': repo_type + }, + { + 'ParameterKey': 'Repository', + 'ParameterValue': repository + } + ], + Capabilities=['CAPABILITY_NAMED_IAM']) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName=stack_name) + logger.info('us-east-1: CodeBuild configuration complete') + + codebuild = session.client('codebuild', region_name='us-east-1') + rel_build = codebuild.start_build( + projectName='perf-test-release-build-' + perf_test_id, + sourceVersion=commit_id)['build']['id'] + prof_build = codebuild.start_build( + projectName='perf-test-profiling-build-' + perf_test_id, + sourceVersion=commit_id)['build']['id'] + + logger.info('us-east-1: Waiting for builds to complete') + all_complete = False + completed = {} + while not all_complete: + time.sleep(10) + response = codebuild.batch_get_builds(ids=[rel_build, prof_build]) + all_complete = True + for b in response['builds']: + if b['buildStatus'] == 'IN_PROGRESS': + all_complete = False + break + elif not completed.get(b['projectName']): + logger.info('us-east-1: Build {} ended: {}'.format( + b['projectName'], b['buildStatus'])) + assert(b['buildStatus'] == 'SUCCEEDED') + completed[b['projectName']] = True + + stack.delete() + waiter = cfn.meta.client.get_waiter('stack_delete_complete') + waiter.wait(StackName=stack_name) + logger.info('us-east-1: CodeBuild complete and stack cleaned') + + +def prepare_sqs(session, region, perf_test_id, tasks): + # create a bucket for storing artifacts + logger = logging.getLogger('perf_test') + + cfn = session.resource('cloudformation', region_name=region) + stack_name = 'perf-test-sqs-' + perf_test_id + with open(same_dir('sqs.yaml')) as f: + CFN_sqs = f.read() + stack = cfn.create_stack( + StackName=stack_name, + TemplateBody=CFN_sqs, + Parameters=[ + { + 'ParameterKey': 'PerfTestId', + 'ParameterValue': perf_test_id + } + ]) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName=stack_name, WaiterConfig={'Delay': 10}) + for o in stack.outputs: + if o['OutputKey'] == 'QueueArn': + arn = o['OutputValue'] + elif o['OutputKey'] == 'QueueName': + queue = o['OutputValue'] + elif o['OutputKey'] == 'QueueUrl': + url = o['OutputValue'] + else: + assert(False) + logger.info(region + ': SQS queues {}, {}-run set up'.format( + queue, queue)) + + seed_queue(session, region, queue, tasks) + + return (queue, arn, url) + + +def seed_queue(session, region, queue, task_set): + # set up the tasks + logger = logging.getLogger('perf_test') + + all_tasks = ['ConcurrencySafety-Main', 'DefinedBehavior-Arrays', + 'DefinedBehavior-TerminCrafted', 'MemSafety-Arrays', + 'MemSafety-Heap', 'MemSafety-LinkedLists', + 'MemSafety-Other', 'MemSafety-TerminCrafted', + 'Overflows-BitVectors', 'Overflows-Other', + 'ReachSafety-Arrays', 'ReachSafety-BitVectors', + 'ReachSafety-ControlFlow', 'ReachSafety-ECA', + 'ReachSafety-Floats', 'ReachSafety-Heap', + 'ReachSafety-Loops', 'ReachSafety-ProductLines', + 'ReachSafety-Recursive', 'ReachSafety-Sequentialized', + 'Systems_BusyBox_MemSafety', 'Systems_BusyBox_Overflows', + 'Systems_DeviceDriversLinux64_ReachSafety', + 'Termination-MainControlFlow', 'Termination-MainHeap', + 'Termination-Other'] + + sqs = session.resource('sqs', region_name=region) + queue = sqs.get_queue_by_name(QueueName=queue) + if task_set == 'full': + tasks = all_tasks + elif task_set == 'quick': + tasks = ['ReachSafety-Loops', 'ReachSafety-BitVectors'] + else: + tasks = [task_set] + + for t in tasks: + assert(t in set(all_tasks)) + + for t in tasks: + response = queue.send_messages( + Entries=[ + {'Id': '1', 'MessageBody': 'release-' + t}, + {'Id': '2', 'MessageBody': 'profiling-' + t} + ]) + assert(not response.get('Failed')) + + +def run_perf_test( + session, mode, region, az, ami, instance_type, sqs_arn, sqs_url, + parallel, snapshot_id, instance_terminated_arn, bucket_name, + perf_test_id, price, ssh_key_name): + # create an EC2 instance and trigger benchmarking + logger = logging.getLogger('perf_test') + + if mode == 'spot': + price = str(price*3) + elif mode == 'on-demand': + price = '' + else: + # Batch not yet implemented + assert(False) + + stack_name = 'perf-test-exec-' + perf_test_id + logger.info(region + ': Creating stack ' + stack_name) + cfn = session.resource('cloudformation', region_name=region) + with open(same_dir('ec2.yaml')) as f: + CFN_ec2 = f.read() + stack = cfn.create_stack( + StackName=stack_name, + TemplateBody=CFN_ec2, + Parameters=[ + { + 'ParameterKey': 'Ami', + 'ParameterValue': ami + }, + { + 'ParameterKey': 'AvailabilityZone', + 'ParameterValue': az + }, + { + 'ParameterKey': 'InstanceType', + 'ParameterValue': instance_type + }, + { + 'ParameterKey': 'SnapshotId', + 'ParameterValue': snapshot_id + }, + { + 'ParameterKey': 'S3Bucket', + 'ParameterValue': bucket_name + }, + { + 'ParameterKey': 'PerfTestId', + 'ParameterValue': perf_test_id + }, + { + 'ParameterKey': 'SnsTopic', + 'ParameterValue': instance_terminated_arn + }, + { + 'ParameterKey': 'SqsArn', + 'ParameterValue': sqs_arn + }, + { + 'ParameterKey': 'SqsUrl', + 'ParameterValue': sqs_url + }, + { + 'ParameterKey': 'MaxPrice', + 'ParameterValue': price + }, + { + 'ParameterKey': 'FleetSize', + 'ParameterValue': str(parallel) + }, + { + 'ParameterKey': 'SSHKeyName', + 'ParameterValue': ssh_key_name + } + ], + Capabilities=['CAPABILITY_NAMED_IAM']) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName=stack_name) + asg_name = stack.outputs[0]['OutputValue'] + asg = session.client('autoscaling', region_name=region) + # make sure hosts that have been shut down don't come back + asg.suspend_processes( + AutoScalingGroupName=asg_name, + ScalingProcesses=['ReplaceUnhealthy']) + while True: + res = asg.describe_auto_scaling_instances() + if len(res['AutoScalingInstances']) == parallel: + break + logger.info(region + ': Waiting for AutoScalingGroup to be populated') + time.sleep(10) + # https://gist.github.com/alertedsnake/4b85ea44481f518cf157 + instances = [a['InstanceId'] for a in res['AutoScalingInstances'] + if a['AutoScalingGroupName'] == asg_name] + + ec2 = session.client('ec2', region_name=region) + for instance_id in instances: + i_res = ec2.describe_instances(InstanceIds=[instance_id]) + name = i_res['Reservations'][0]['Instances'][0]['PublicDnsName'] + logger.info(region + ': Running benchmarks on ' + name) + + +def main(): + logging_format = "%(asctime)-15s: %(message)s" + logging.basicConfig(format=logging_format) + logger = logging.getLogger('perf_test') + logger.setLevel('DEBUG') + + args = parse_args() + + # pick the most suitable region + session = boto3.session.Session() + (region, az, price, ami) = select_region( + session, args.mode, args.region, args.instance_type) + + # fail early if key configuration would fail + if args.ssh_key_name: + ec2 = session.client('ec2', region_name=region) + res = ec2.describe_key_pairs( + Filters=[ + {'Name': 'key-name', 'Values': [args.ssh_key_name]} + ]) + if not args.ssh_key: + assert(len(res['KeyPairs']) == 1) + elif len(res['KeyPairs']): + logger.warning(region + ': Key pair "' + args.ssh_key_name + + '" already exists, ignoring key material') + else: + with open(args.ssh_key) as kf: + pk = kf.read() + ec2.import_key_pair( + KeyName=args.ssh_key_name, PublicKeyMaterial=pk) + + # build a unique id for this performance test run + perf_test_id = str(datetime.datetime.utcnow().isoformat( + sep='-', timespec='seconds')) + '-' + args.commit_id + perf_test_id = re.sub('[:/_\.\^~ ]', '-', perf_test_id) + logger.info('global: Preparing performance test ' + perf_test_id) + + # target storage name + account_id = session.client('sts').get_caller_identity()['Account'] + bucket_name = "perf-test-" + account_id + + # configuration set, let's create the infrastructure + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as e: + session1 = boto3.session.Session() + sns_s3_future = e.submit( + prepare_sns_s3, session1, args.email, bucket_name) + session2 = boto3.session.Session() + build_future = e.submit( + build, session2, args.repository, args.commit_id, bucket_name, + perf_test_id) + session3 = boto3.session.Session() + ebs_future = e.submit(prepare_ebs, session3, region, az, ami) + session4 = boto3.session.Session() + sqs_future = e.submit( + prepare_sqs, session4, region, perf_test_id, args.tasks) + + # wait for all preparation steps to complete + instance_terminated_arn = sns_s3_future.result() + build_future.result() + snapshot_id = ebs_future.result() + (queue, sqs_arn, sqs_url) = sqs_future.result() + + run_perf_test( + session, args.mode, region, az, ami, args.instance_type, + sqs_arn, sqs_url, args.parallel, snapshot_id, + instance_terminated_arn, bucket_name, perf_test_id, price, + args.ssh_key_name) + + return 0 + + +if __name__ == '__main__': + rc = main() + sys.exit(rc) diff --git a/scripts/perf-test/s3.yaml b/scripts/perf-test/s3.yaml new file mode 100644 index 00000000000..d8c7a8b5a79 --- /dev/null +++ b/scripts/perf-test/s3.yaml @@ -0,0 +1,38 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + SnsTopicArn: + Type: String + + S3BucketName: + Type: String + +Resources: + SnsTopic: + Type: "AWS::SNS::TopicPolicy" + Properties: + Topics: + - !Ref SnsTopicArn + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - sns:Publish + Principal: + AWS: "*" + Resource: !Ref SnsTopicArn + Condition: + ArnLike: + AWS:SourceArn: !Sub "arn:aws:s3:::${S3BucketName}" + + S3Bucket: + DependsOn: SnsTopic + Type: "AWS::S3::Bucket" + Properties: + BucketName: !Ref S3BucketName + NotificationConfiguration: + TopicConfigurations: + - Event: s3:ObjectCreated:* + Topic: !Ref SnsTopicArn diff --git a/scripts/perf-test/sns.yaml b/scripts/perf-test/sns.yaml new file mode 100644 index 00000000000..a51899074c1 --- /dev/null +++ b/scripts/perf-test/sns.yaml @@ -0,0 +1,27 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + NotificationAddress: + Type: String + + SnsTopicName: + Type: String + +Resources: + SnsTopic: + Type: "AWS::SNS::Topic" + Properties: + DisplayName: !Ref SnsTopicName + TopicName: !Ref SnsTopicName + + SnsSubscription: + Type: "AWS::SNS::Subscription" + Properties: + Endpoint: !Ref NotificationAddress + Protocol: email + TopicArn: !Ref SnsTopic + +Outputs: + TopicArn: + Value: !Ref SnsTopic diff --git a/scripts/perf-test/sqs.yaml b/scripts/perf-test/sqs.yaml new file mode 100644 index 00000000000..11d6d87a1da --- /dev/null +++ b/scripts/perf-test/sqs.yaml @@ -0,0 +1,27 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + PerfTestId: + Type: String + +Resources: + Queue: + Type: "AWS::SQS::Queue" + Properties: + QueueName: !Sub "perf-test-${PerfTestId}" + + QueueDone: + Type: "AWS::SQS::Queue" + Properties: + QueueName: !Sub "perf-test-${PerfTestId}-run" + +Outputs: + QueueName: + Value: !GetAtt Queue.QueueName + + QueueUrl: + Value: !Ref Queue + + QueueArn: + Value: !GetAtt Queue.Arn