Skip to content

Commit

Permalink
Merge branch 'release-1.3.23'
Browse files Browse the repository at this point in the history
* release-1.3.23:
  Bumping version to 1.3.23
  Update changelog with latest changes
  EMR: changed to use regionalized buckets.
  EMR: Support installing hive-site.xml in create-cluster and install-applications commands.
  Update changelog with bugfix from boto/botocore#321
  Add #824 to the changelog
  Rename of elasticloadbalancing -> elb, updates from docteam
  Optional s3 sync flag to address timestamp issues described in #599
  • Loading branch information
jamesls committed Jul 16, 2014
2 parents 0acbafd + edf6eb4 commit 7526a9d
Show file tree
Hide file tree
Showing 42 changed files with 443 additions and 189 deletions.
17 changes: 17 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,23 @@ CHANGELOG
=========


1.3.23
======

* feature:``aws support``: Update ``aws support`` command to
the latest version
* feature:``aws iam``: Update ``aws iam`` command to the latest
version
* feature:``aws emr``: Add ``--hive-site`` option to
``aws emr create-cluster`` and ``aws emr install-application`` commands
* feature:``aws s3 sync``: Add an ``--exact-timestamps`` option
to the ``aws s3 sync`` command
(`issue 824 <https://github.com/aws/aws-cli/pull/824>`__)
* bugfix:``aws ec2 copy-snapshot``: Fix bug when spaces in
the description caused the copy request to fail
(`issue botocore 321 <https://github.com/boto/botocore/pull/321>`__)


1.3.22
======

Expand Down
2 changes: 1 addition & 1 deletion awscli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"""
import os

__version__ = '1.3.22'
__version__ = '1.3.23'

#
# Get our data path to be added to botocore's search path
Expand Down
57 changes: 55 additions & 2 deletions awscli/customizations/emr/applicationutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,19 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None):
if hive_version is None:
hive_version = constants.LATEST
step_list.append(
emrutils.build_hive_install_step(
_build_install_hive_step(
region=parsed_globals.region,
version=hive_version))
args = app_config.get('Args')
if args is not None:
hive_site_path = _find_matching_arg(
key=constants.HIVE_SITE_KEY, args_list=args)
if hive_site_path is not None:
step_list.append(
_build_install_hive_site_step(
region=parsed_globals.region,
version=hive_version,
hive_site_path=hive_site_path))
elif app_name == constants.PIG:
pig_version = app_config.get('Version')
if pig_version is None:
Expand Down Expand Up @@ -122,4 +132,47 @@ def build_impala_install_bootstrap_action(region, version, args=None):
path=emrutils.build_s3_link(
relative_path=constants.IMPALA_INSTALL_PATH,
region=region),
args=args_list)
args=args_list)


def _build_install_hive_step(region, version,
action_on_failure=constants.TERMINATE_CLUSTER):
step_args = [
emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
constants.INSTALL_HIVE_ARG,
constants.BASE_PATH_ARG,
emrutils.build_s3_link(constants.HIVE_BASE_PATH),
constants.HIVE_VERSIONS,
version]
step = emrutils.build_step(
name=constants.INSTALL_HIVE_NAME,
action_on_failure=action_on_failure,
jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
args=step_args)
return step


def _build_install_hive_site_step(region, version, hive_site_path,
action_on_failure=constants.CANCEL_AND_WAIT):
step_args = [
emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
constants.BASE_PATH_ARG,
emrutils.build_s3_link(constants.HIVE_BASE_PATH),
constants.INSTALL_HIVE_SITE_ARG,
hive_site_path,
constants.HIVE_VERSIONS,
version]
step = emrutils.build_step(
name=constants.INSTALL_HIVE_SITE_NAME,
action_on_failure=action_on_failure,
jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
args=step_args)
return step


def _find_matching_arg(key, args_list):
for arg in args_list:
if key in arg:
return arg

return None
3 changes: 3 additions & 0 deletions awscli/customizations/emr/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@
INSTALL_PIG_NAME = 'Install Pig'
INSTALL_HIVE_ARG = '--install-hive'
INSTALL_HIVE_NAME = 'Install Hive'
HIVE_SITE_KEY = '--hive-site'
INSTALL_HIVE_SITE_ARG = '--install-hive-site'
INSTALL_HIVE_SITE_NAME = 'Install Hive Site Configuration'
BASE_PATH_ARG = '--base-path'
INSTALL_GANGLIA_NAME = 'Install Ganglia'
INSTALL_HBASE_NAME = 'Install HBase'
Expand Down
30 changes: 7 additions & 23 deletions awscli/customizations/emr/emrutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,15 @@ def build_bootstrap_action(
return ba_config


def build_s3_link(relative_path='', region=None):
if region and region != 'us-east-1':
return 's3://{0}.elasticmapreduce{1}'.format(region, relative_path)
else:
return 's3://elasticmapreduce{0}'.format(relative_path)
def build_s3_link(relative_path='', region='us-east-1'):
if region is None:
region = 'us-east-1'
return 's3://{0}.elasticmapreduce{1}'.format(region, relative_path)


def get_script_runner(region=None):
def get_script_runner(region='us-east-1'):
if region is None:
region = 'us-east-1'
return build_s3_link(
relative_path=constants.SCRIPT_RUNNER_PATH, region=region)

Expand Down Expand Up @@ -164,23 +165,6 @@ def build_pig_install_step(region, version,
return step


def build_hive_install_step(region, version,
action_on_failure=constants.TERMINATE_CLUSTER):
step_args = [
build_s3_link(constants.HIVE_SCRIPT_PATH, region),
constants.INSTALL_HIVE_ARG,
constants.BASE_PATH_ARG,
build_s3_link(constants.HIVE_BASE_PATH),
constants.HIVE_VERSIONS,
version]
step = build_step(
name=constants.INSTALL_HIVE_NAME,
action_on_failure=action_on_failure,
jar=build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
args=step_args)
return step


def call(session, operation_object, parameters, region_name=None,
endpoint_url=None, verify=None):
# We could get an error from get_endpoint() about not having
Expand Down
9 changes: 9 additions & 0 deletions awscli/customizations/s3/comparator.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ def __init__(self, params=None):
if 'size_only' in params:
self.compare_on_size_only = params['size_only']

self.match_exact_timestamps = False
if 'exact_timestamps' in params:
self.match_exact_timestamps = params['exact_timestamps']

def call(self, src_files, dest_files):
"""
This function preforms the actual comparisons. The parameters it takes
Expand Down Expand Up @@ -198,6 +202,11 @@ def compare_time(self, src_file, dest_file):
# at the source location.
return False
elif cmd == "download":
if self.match_exact_timestamps:
# An update is needed unless the
# timestamps match exactly.
return total_seconds(delta) == 0

if total_seconds(delta) <= 0:
return True
else:
Expand Down
6 changes: 6 additions & 0 deletions awscli/customizations/s3/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,7 @@ def add_verify_ssl(self, parsed_globals):
'sse', 'storage-class', 'content-type',
'cache-control', 'content-disposition',
'content-encoding', 'content-language',
'exact-timestamps',
'expires', 'size-only']},
'ls': {'options': {'nargs': '?', 'default': 's3://'},
'params': ['recursive'], 'default': 's3://',
Expand Down Expand Up @@ -867,6 +868,11 @@ def add_verify_ssl(self, parsed_globals):
'size-only': {'options': {'action': 'store_true'}, 'documents':
('Makes the size of each key the only criteria used to '
'decide whether to sync from source to destination.')},
'exact-timestamps': {'options': {'action': 'store_true'}, 'documents':
('When syncing from S3 to local, same-sized items will be '
'ignored only when the timestamps match exactly. The '
'default behavior is to ignore same-sized items unless '
'the local version is newer than the S3 version.')},
'index-document': {'options': {}, 'documents':
('A suffix that is appended to a request that is for a '
'directory on the website endpoint (e.g. if the suffix '
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ The following example creates a scheduled action to scale on a recurring schedul

aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --recurrence "30 0 1 1,6,12 0" --min-size 2 --max-size 6 --desired-capacity 4

For more information, see `Scheduled Scaling`_ in the *Auto Scaling Developer Guide*.
For more information, see `Scheduled Scaling`__ in the *Auto Scaling Developer Guide*.

.. _`Scheduled Scaling`: http://docs.aws.amazon.com/AutoScaling/latest/DeveloperGuide/schedule_time.html
.. __: http://docs.aws.amazon.com/AutoScaling/latest/DeveloperGuide/schedule_time.html

22 changes: 11 additions & 11 deletions awscli/examples/emr/create-cluster-examples.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
- Command::

aws emr create-cluster --ami-version 3.1.0 --service-role EMR_DefaultRole --ec2-attributes InstanceProfiles=EC2_EMR_DefaultRoles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

**3. Create an Amazon EMR cluster with default roles
- Command::
Expand All @@ -20,31 +20,31 @@

- Command::

aws emr create-cluster --ami-version 3.1.0 --auto-terminate --instance-groups Name=Master,InstanceGroupType=MASTER,InstanceType=m3.xlarge,InstanceCount=1 Name=Core,InstanceGroupType=CORE,InstanceType=m3.xlarge,InstanceCount=2 Name=Task,InstanceGroupType=TASK,InstanceType=m3.xlarge,InstanceCount=2
aws emr create-cluster --ami-version 3.1.0 --auto-terminate --instance-groups Name=Master,InstanceGroupType=MASTER,InstanceType=m3.xlarge,InstanceCount=1 Name=Core,InstanceGroupType=CORE,InstanceType=m3.xlarge,InstanceCount=2 Name=Task,InstanceGroupType=TASK,InstanceType=m3.xlarge,InstanceCount=2

**5. Specify whether the cluster should terminate after completing all the steps**

- Create an Amazon EMR cluster that will terminate after completing all the steps::

aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

- Create an Amazon EMR cluster that will NOT terminate after completing all the steps::

aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --no-auto-terminate
aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --no-auto-terminate

**6. Specify EC2 Attributes**

- Create an Amazon EMR cluster with Amazon EC2 Key Pair "myKey" and instance profile "myProfile"::

aws emr create-cluster --ec2-attributes KeyName=myKey,InstanceProfile=myRole --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
aws emr create-cluster --ec2-attributes KeyName=myKey,InstanceProfile=myRole --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

- Create an Amazon EMR cluster in an Amazon VPC subnet::

aws emr create-cluster --ec2-attributes SubnetId=subnet-xxxxx --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
aws emr create-cluster --ec2-attributes SubnetId=subnet-xxxxx --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

- Create an Amazon EMR cluster in an AvailabilityZone. For example, us-west-1b::

aws emr create-cluster --ec2-attributes AvailabilityZone=us-west-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
aws emr create-cluster --ec2-attributes AvailabilityZone=us-west-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

**7. Enable debugging and specify a Log URI**

Expand All @@ -56,7 +56,7 @@

- Add a list of tags::

aws emr create-cluster --tags name="John Doe" age=29 address="123 East NW Seattle" --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
aws emr create-cluster --tags name="John Doe" age=29 address="123 East NW Seattle" --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

- List tags of an Amazon EMR cluster::

Expand All @@ -66,7 +66,7 @@

- Command::

aws emr create-cluster --bootstrap-actions Path=s3://mybucket/myscript1,Name=BootstrapAction1,Args=[arg1,arg2] Path=s3://mybucket/myscript2,Name=BootstrapAction2,Args=[arg1,arg2] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
aws emr create-cluster --bootstrap-actions Path=s3://mybucket/myscript1,Name=BootstrapAction1,Args=[arg1,arg2] Path=s3://mybucket/myscript2,Name=BootstrapAction2,Args=[arg1,arg2] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

- The following example changes the maximum number of map tasks and sets the NameNode heap size::

Expand All @@ -76,7 +76,7 @@

- Create an Amazon EMR cluster with Hive, Pig, HBase, Ganglia, and Impala installed::

aws emr create-cluster --applications Name=Hive Name=Pig Name=HBase Name=Ganglia Name=Impala,Args=[IMPALA_BACKEND_PORT=22001,IMPALA_MEM_LIMIT=70%] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
aws emr create-cluster --applications Name=Hive Name=Pig Name=HBase Name=Ganglia Name=Impala,Args=[IMPALA_BACKEND_PORT=22001,IMPALA_MEM_LIMIT=70%] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

- Create an Amazon EMR cluster with Hive and Pig installed::

Expand Down Expand Up @@ -109,7 +109,7 @@
**13. To add Streaming steps when creating an Amazon EMR cluster**

- Command::

aws emr create-cluster --steps Type=STREAMING,Name='Streaming Program',ActionOnFailure=CONTINUE,Args=-mapper,mymapper,-reducer,myreducer,-input,myinput,-output,myoutput Type=STREAMING,Name='Streaming Program',ActionOnFailure=CONTINUE,Args=--files,s3://elasticmapreduce/samples/wordcount/wordSplitter.py,-mapper,wordSplitter.py,-reducer,aggregate,-input,s3://elasticmapreduce/samples/wordcount/input,-output,s3://mybucket/wordcount/output --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate

- Streaming steps required parameters::
Expand Down
4 changes: 2 additions & 2 deletions awscli/examples/emr/create-cluster-synopsis.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
--instance-groups <value>
--auto-terminate | --no-auto-terminate
[--use-default-role]
[--service-role <value>]
[--service-role <value>]
[--name <value>]
[--log-uri <value>]
[--additional-info <value>]
Expand All @@ -15,4 +15,4 @@
[--applications <value>]
[--bootstrap-actions <value>]
[--steps <value>]
[--restore-from-hbase-backup <value>]
[--restore-from-hbase-backup <value>]
Loading

0 comments on commit 7526a9d

Please sign in to comment.