8000 Adding support for EMR resources by alonsodomin · Pull Request #421 · cloudtools/troposphere · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Adding support for EMR resources #421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 30 commits into from
Apr 4, 2016
Merged

Adding support for EMR resources #421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
9cedf60
Initial files for EMR support
Mar 3, 2016
7bce048
Adding some tests for EMR types
Mar 3, 2016
0deac41
Reformatting the file
Mar 4, 2016
d7d2791
Adding a basic sample of an EMR Cluster plus the expected template
Mar 5, 2016
a6fe775
Fixing formatting
Mar 5, 2016
8af50f9
Fixing print syntax
Mar 5, 2016
a6c8588
Adding model for ScriptBootstrapActions
Mar 8, 2016
0c491a1
Adding configuration properties to the cluster
Mar 8, 2016
4ec95b5
Using helper function for colon-separated key-value pair lists.
Mar 8, 2016
06fccad
Fixing formatting
Mar 8, 2016
5a75661
Setting a type for the configuration list in instance group resource
Mar 8, 2016
dd92a66
Added additional validators and aligned fixed test template
Mar 8, 2016
e6f5013
Added a Test step into the test template.
Mar 8, 2016
d6ac3a3
Initial files for EMR support
Mar 3, 2016
288acc9
Adding some tests for EMR types
Mar 3, 2016
c34b755
Reformatting the file
Mar 4, 2016
a8a1fe1
Adding a basic sample of an EMR Cluster plus the expected template
Mar 5, 2016
9d1f7bd Fixing formatting
Mar 5, 2016
b67cb0d
Fixing print syntax
Mar 5, 2016
7570bef
Adding model for ScriptBootstrapActions
Mar 8, 2016
f401cd5
Adding configuration properties to the cluster
Mar 8, 2016
44fcfb7
Using helper function for colon-separated key-value pair lists.
Mar 8, 2016
e271119
Fixing formatting
Mar 8, 2016
ab58880
Setting a type for the configuration list in instance group resource
Mar 8, 2016
a9af32a
Added additional validators and aligned fixed test template
Mar 8, 2016
d2486b7
Added a Test step into the test template.
Mar 8, 2016
3e8a15c
Update validators for security groups and instance count
Mar 17, 2016
bddd616
Move StepProperties to the appropiate type
Mar 17, 2016
1205091
Fix the type for Application args and add step properties to the test…
Mar 17, 2016
c2bdd62
Merge branch 'master' of https://github.com/alonsodomin/troposphere
Mar 24, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 170 additions & 0 deletions examples/EMR_Cluster.py
View file
Open in desktop
10000
Original file line number Diff line numberDiff line change
@@ -0,0 +1,170 @@
from troposphere import Parameter, Ref, Template, Tags
import troposphere.iam as iam
import troposphere.emr as emr

template = Template()
template.add_description(
"Sample CloudFormation template for creating an EMR cluster"
)

keyname = template.add_parameter(Parameter(
"KeyName",
Description="Name of an existing EC2 KeyPair to enable SSH "
"to the instances",
Type="AWS::EC2::KeyPair::KeyName"
))

subnet = template.add_parameter(Parameter(
"Subnet",
Description="Subnet ID for creating the EMR cluster",
Type="AWS::EC2::Subnet::Id"
))

service_access_sg = template.add_parameter(Parameter(
"ServiceAccessSecurityGroup",
Description="Security Group providing service access to EMR",
Type="AWS::EC2::SecurityGroup::Id"
))

managed_master_sg = template.add_parameter(Parameter(
"ManagedMasterSecurityGroup",
Description="Security Group (managed by EMR) for master instances",
Type="AWS::EC2::SecurityGroup::Id"
))

managed_slave_sq = template.add_parameter(Parameter(
"ManagedSlaveSecurityGroup",
Description="Security Group (managed by EMR) for slave instances",
Type="AWS::EC2::SecurityGroup::Id"
))

# IAM roles required by EMR

emr_service_role = template.add_resource(iam.Role(
'EMRServiceRole',
AssumeRolePolicyDocument={
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": [
"elasticmapreduce.amazonaws.com"
]
},
"Action": ["sts:AssumeRole"]
}]
},
ManagedPolicyArns=[
'arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole'
]
))

emr_job_flow_role = template.add_resource(iam.Role(
"EMRJobFlowRole",
AssumeRolePolicyDocument={
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": [
"ec2.amazonaws.com"
]
},
"Action": ["sts:AssumeRole"]
}]
},
ManagedPolicyArns=[
'arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role'
]
))

emr_instance_profile = template.add_resource(iam.InstanceProfile(
"EMRInstanceProfile",
Roles=[Ref(emr_job_flow_role)]
))

# EMR Cluster Resource

cluster = template.add_resource(emr.Cluster(
"EMRSampleCluster",
Name="EMR Sample Cluster",
ReleaseLabel='emr-4.3.0',
BootstrapActions=[emr.BootstrapActionConfig(
Name='Dummy bootstrap action',
ScriptBootstrapAction=emr.ScriptBootstrapActionConfig(
Path='/bin/sh',
Args=['echo', 'Hello World']
)
)],
Configurations=[
emr.Configuration(
Classification="core-site",
ConfigurationProperties={
'hadoop.security.groups.cache.secs': '250'
}
),
emr.Configuration(
Classification="mapred-site",
ConfigurationProperties={
'mapred.tasktracker.map.tasks.maximum': '2',
'mapreduce.map.sort.spill.percent': '90',
'mapreduce.tasktracker.reduce.tasks.maximum': '5'
}
),
emr.Configuration(
Classification="hadoop-env",
Configurations=[
emr.Configuration(
Classification="export",
ConfigurationProperties={
"HADOOP_DATANODE_HEAPSIZE": "2048",
"HADOOP_NAMENODE_OPTS": "-XX:GCTimeRatio=19"
}
)
]
)
],
JobFlowRole=Ref(emr_instance_profile),
ServiceRole=Ref(emr_service_role),
Instances=emr.JobFlowInstancesConfig(
MasterInstanceGroup=emr.InstanceGroupConfigProperty(
Name="Master Instance",
InstanceCount="1",
InstanceType="m3.xlarge",
Market="ON_DEMAND"
),
CoreInstanceGroup=emr.InstanceGroupConfigProperty(
Name="Core Instance",
BidPrice="20",
InstanceCount="1",
InstanceType="m3.xlarge",
Market="SPOT"
)
),
Applications=[
emr.Application(Name="Hadoop"),
emr.Application(Name="Hive"),
emr.Application(Name="Mahout"),
emr.Application(Name="Pig"),
emr.Application(Name="Spark")
],
VisibleToAllUsers="true",
Tags=Tags(
Name="EMR Sample Cluster"
)
))

step = template.add_resource(emr.Step(
'TestStep',
Name="TestStep",
ActionOnFailure='CONTINUE',
HadoopJarStep=emr.HadoopJarStepConfig(
Args=["5", "10"],
Jar="s3://emr-cfn-test/hadoop-mapreduce-examples-2.6.0.jar",
MainClass="pi",
StepProperties=[
emr.KeyValue('my.custom.property', 'my.custom.value')
]
),
JobFlowId=Ref(cluster)
))

print(template.to_json())
199 changes: 199 additions & 0 deletions tests/examples_output/EMR_Cluster.template
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
{
"Description": "Sample CloudFormation template for creating an EMR cluster",
"Parameters": {
"KeyName": {
"Description": "Name of an existing EC2 KeyPair to enable SSH to the instances",
"Type": "AWS::EC2::KeyPair::KeyName"
},
"ManagedMasterSecurityGroup": {
"Description": "Security Group (managed by EMR) for master instances",
"Type": "AWS::EC2::SecurityGroup::Id"
},
"ManagedSlaveSecurityGroup": {
"Description": "Security Group (managed by EMR) for slave instances",
"Type": "AWS::EC2::SecurityGroup::Id"
},
"ServiceAccessSecurityGroup": {
"Description": "Security Group providing service access to EMR",
"Type": "AWS::EC2::SecurityGroup::Id"
},
"Subnet": {
"Description": "Subnet ID for creating the EMR cluster",
"Type": "AWS::EC2::Subnet::Id"
}
},
"Resources": {
"EMRInstanceProfile": {
"Properties": {
"Roles": [
{
"Ref": "EMRJobFlowRole"
}
]
},
"Type": "AWS::IAM::InstanceProfile"
},
"EMRJobFlowRole": {
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": [
"sts:AssumeRole"
],
"Effect": "Allow",
"Principal": {
"Service": [
"ec2.amazonaws.com"
]
}
}
]
},
"ManagedPolicyArns": [
"arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
]
},
"Type": "AWS::IAM::Role"
},
"EMRSampleCluster": {
"Properties": {
"Applications": [
{
"Name": "Hadoop"
},
{
"Name": "Hive"
},
{
"Name": "Mahout"
},
{
"Name": "Pig"
},
{
"Name": "Spark"
}
],
"BootstrapActions": [
{
"Name": "Dummy bootstrap action",
"ScriptBootstrapAction": {
"Args": [
"echo",
"Hello World"
],
"Path": "/bin/sh"
}
}
],
"Configurations": [
{
"Classification": "core-site",
"ConfigurationProperties": {
"hadoop.security.groups.cache.secs": "250"
}
},
{
"Classification": "mapred-site",
"ConfigurationProperties": {
"mapred.tasktracker.map.tasks.maximum": "2",
"mapreduce.map.sort.spill.percent": "90",
"mapreduce.tasktracker.reduce.tasks.maximum": "5"
}
},
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"ConfigurationProperties": {
"HADOOP_DATANODE_HEAPSIZE": "2048",
"HADOOP_NAMENODE_OPTS": "-XX:GCTimeRatio=19"
}
}
]
}
],
"Instances": {
"CoreInstanceGroup": {
"BidPrice": "20",
"InstanceCount": "1",
"InstanceType": "m3.xlarge",
"Market": "SPOT",
"Name": "Core Instance"
},
"MasterInstanceGroup": {
"InstanceCount": "1",
"InstanceType": "m3.xlarge",
"Market": "ON_DEMAND",
"Name": "Master Instance"
}
},
"JobFlowRole": {
"Ref": "EMRInstanceProfile"
},
"Name": "EMR Sample Cluster",
"ReleaseLabel": "emr-4.3.0",
"ServiceRole": {
"Ref": "EMRServiceRole"
},
"Tags": [
{
"Key": "Name",
"Value": "EMR Sample Cluster"
}
],
"VisibleToAllUsers": "true"
},
"Type": "AWS::EMR::Cluster"
},
"EMRServiceRole": {
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": [
"sts:AssumeRole"
],
"Effect": "Allow",
"Principal": {
"Service": [
"elasticmapreduce.amazonaws.com"
]
}
}
]
},
"ManagedPolicyArns": [
"arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole"
]
},
"Type": "AWS::IAM::Role"
},
"TestStep": {
"Properties": {
"ActionOnFailure": "CONTINUE",
"HadoopJarStep": {
"Args": [
"5",
"10"
],
"Jar": "s3://emr-cfn-test/hadoop-mapreduce-examples-2.6.0.jar",
"MainClass": "pi",
"StepProperties": [
{
"Key": "my.custom.property",
"Value": "my.custom.value"
}
]
},
"JobFlowId": {
"Ref": "EMRSampleCluster"
},
"Name": "TestStep"
},
"Type": "AWS::EMR::Step"
}
}
}
Loading
0