| OLD | NEW |
|---|
| (Empty) | |
| 1 # Copyright (c) 2010 Jeremy Thurgood <firxen+boto@gmail.com> |
| 2 # |
| 3 # Permission is hereby granted, free of charge, to any person obtaining a |
| 4 # copy of this software and associated documentation files (the |
| 5 # "Software"), to deal in the Software without restriction, including |
| 6 # without limitation the rights to use, copy, modify, merge, publish, dis- |
| 7 # tribute, sublicense, and/or sell copies of the Software, and to permit |
| 8 # persons to whom the Software is furnished to do so, subject to the fol- |
| 9 # lowing conditions: |
| 10 # |
| 11 # The above copyright notice and this permission notice shall be included |
| 12 # in all copies or substantial portions of the Software. |
| 13 # |
| 14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
| 15 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- |
| 16 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT |
| 17 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, |
| 18 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 19 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| 20 # IN THE SOFTWARE. |
| 21 |
| 22 |
| 23 # NOTE: These tests only cover the very simple cases I needed to test |
| 24 # for the InstanceGroup fix. |
| 25 |
| 26 import xml.sax |
| 27 import unittest |
| 28 |
| 29 from boto import handler |
| 30 from boto.emr import emrobject |
| 31 from boto.resultset import ResultSet |
| 32 |
| 33 |
| 34 JOB_FLOW_EXAMPLE = """ |
| 35 <DescribeJobFlowsResponse |
| 36 xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-01-15"> |
| 37 <DescribeJobFlowsResult> |
| 38 <JobFlows> |
| 39 <member> |
| 40 <ExecutionStatusDetail> |
| 41 <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime> |
| 42 <StartDateTime>2009-01-28T21:49:16Z</StartDateTime> |
| 43 <State>STARTING</State> |
| 44 </ExecutionStatusDetail> |
| 45 <Name>MyJobFlowName</Name> |
| 46 <LogUri>mybucket/subdir/</LogUri> |
| 47 <Steps> |
| 48 <member> |
| 49 <ExecutionStatusDetail> |
| 50 <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime> |
| 51 <State>PENDING</State> |
| 52 </ExecutionStatusDetail> |
| 53 <StepConfig> |
| 54 <HadoopJarStep> |
| 55 <Jar>MyJarFile</Jar> |
| 56 <MainClass>MyMailClass</MainClass> |
| 57 <Args> |
| 58 <member>arg1</member> |
| 59 <member>arg2</member> |
| 60 </Args> |
| 61 <Properties/> |
| 62 </HadoopJarStep> |
| 63 <Name>MyStepName</Name> |
| 64 <ActionOnFailure>CONTINUE</ActionOnFailure> |
| 65 </StepConfig> |
| 66 </member> |
| 67 </Steps> |
| 68 <JobFlowId>j-3UN6WX5RRO2AG</JobFlowId> |
| 69 <Instances> |
| 70 <Placement> |
| 71 <AvailabilityZone>us-east-1a</AvailabilityZone> |
| 72 </Placement> |
| 73 <SlaveInstanceType>m1.small</SlaveInstanceType> |
| 74 <MasterInstanceType>m1.small</MasterInstanceType> |
| 75 <Ec2KeyName>myec2keyname</Ec2KeyName> |
| 76 <InstanceCount>4</InstanceCount> |
| 77 <KeepJobFlowAliveWhenNoSteps>true</KeepJobFlowAliveWhenNoSteps> |
| 78 </Instances> |
| 79 </member> |
| 80 </JobFlows> |
| 81 </DescribeJobFlowsResult> |
| 82 <ResponseMetadata> |
| 83 <RequestId>9cea3229-ed85-11dd-9877-6fad448a8419</RequestId> |
| 84 </ResponseMetadata> |
| 85 </DescribeJobFlowsResponse> |
| 86 """ |
| 87 |
| 88 JOB_FLOW_COMPLETED = """ |
| 89 <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-
03-31"> |
| 90 <DescribeJobFlowsResult> |
| 91 <JobFlows> |
| 92 <member> |
| 93 <ExecutionStatusDetail> |
| 94 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 95 <LastStateChangeReason>Steps completed</LastStateChangeReason> |
| 96 <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> |
| 97 <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime> |
| 98 <State>COMPLETED</State> |
| 99 <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> |
| 100 </ExecutionStatusDetail> |
| 101 <BootstrapActions/> |
| 102 <Name>RealJobFlowName</Name> |
| 103 <LogUri>s3n://example.emrtest.scripts/jobflow_logs/</LogUri> |
| 104 <Steps> |
| 105 <member> |
| 106 <StepConfig> |
| 107 <HadoopJarStep> |
| 108 <Jar>s3n://us-east-1.elasticmapreduce/libs/script-runner/script-
runner.jar</Jar> |
| 109 <Args> |
| 110 <member>s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1
/fetch</member> |
| 111 </Args> |
| 112 <Properties/> |
| 113 </HadoopJarStep> |
| 114 <Name>Setup Hadoop Debugging</Name> |
| 115 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> |
| 116 </StepConfig> |
| 117 <ExecutionStatusDetail> |
| 118 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 119 <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> |
| 120 <State>COMPLETED</State> |
| 121 <EndDateTime>2010-10-21T01:04:22Z</EndDateTime> |
| 122 </ExecutionStatusDetail> |
| 123 </member> |
| 124 <member> |
| 125 <StepConfig> |
| 126 <HadoopJarStep> |
| 127 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</J
ar> |
| 128 <Args> |
| 129 <member>-mapper</member> |
| 130 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-Initial
Mapper.py</member> |
| 131 <member>-reducer</member> |
| 132 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-Initial
Reducer.py</member> |
| 133 <member>-input</member> |
| 134 <member>s3://example.emrtest.data/raw/2010/10/20/*</member> |
| 135 <member>-input</member> |
| 136 <member>s3://example.emrtest.data/raw/2010/10/19/*</member> |
| 137 <member>-input</member> |
| 138 <member>s3://example.emrtest.data/raw/2010/10/18/*</member> |
| 139 <member>-input</member> |
| 140 <member>s3://example.emrtest.data/raw/2010/10/17/*</member> |
| 141 <member>-input</member> |
| 142 <member>s3://example.emrtest.data/raw/2010/10/16/*</member> |
| 143 <member>-input</member> |
| 144 <member>s3://example.emrtest.data/raw/2010/10/15/*</member> |
| 145 <member>-input</member> |
| 146 <member>s3://example.emrtest.data/raw/2010/10/14/*</member> |
| 147 <member>-output</member> |
| 148 <member>s3://example.emrtest.crunched/</member> |
| 149 </Args> |
| 150 <Properties/> |
| 151 </HadoopJarStep> |
| 152 <Name>testjob_Initial</Name> |
| 153 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> |
| 154 </StepConfig> |
| 155 <ExecutionStatusDetail> |
| 156 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 157 <StartDateTime>2010-10-21T01:04:22Z</StartDateTime> |
| 158 <State>COMPLETED</State> |
| 159 <EndDateTime>2010-10-21T01:36:18Z</EndDateTime> |
| 160 </ExecutionStatusDetail> |
| 161 </member> |
| 162 <member> |
| 163 <StepConfig> |
| 164 <HadoopJarStep> |
| 165 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</J
ar> |
| 166 <Args> |
| 167 <member>-mapper</member> |
| 168 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Ma
pper.py</member> |
| 169 <member>-reducer</member> |
| 170 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Re
ducer.py</member> |
| 171 <member>-input</member> |
| 172 <member>s3://example.emrtest.crunched/*</member> |
| 173 <member>-output</member> |
| 174 <member>s3://example.emrtest.step1/</member> |
| 175 </Args> |
| 176 <Properties/> |
| 177 </HadoopJarStep> |
| 178 <Name>testjob_step1</Name> |
| 179 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> |
| 180 </StepConfig> |
| 181 <ExecutionStatusDetail> |
| 182 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 183 <StartDateTime>2010-10-21T01:36:18Z</StartDateTime> |
| 184 <State>COMPLETED</State> |
| 185 <EndDateTime>2010-10-21T01:37:51Z</EndDateTime> |
| 186 </ExecutionStatusDetail> |
| 187 </member> |
| 188 <member> |
| 189 <StepConfig> |
| 190 <HadoopJarStep> |
| 191 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</J
ar> |
| 192 <Args> |
| 193 <member>-mapper</member> |
| 194 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Ma
pper.py</member> |
| 195 <member>-reducer</member> |
| 196 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Re
ducer.py</member> |
| 197 <member>-input</member> |
| 198 <member>s3://example.emrtest.crunched/*</member> |
| 199 <member>-output</member> |
| 200 <member>s3://example.emrtest.step2/</member> |
| 201 </Args> |
| 202 <Properties/> |
| 203 </HadoopJarStep> |
| 204 <Name>testjob_step2</Name> |
| 205 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> |
| 206 </StepConfig> |
| 207 <ExecutionStatusDetail> |
| 208 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 209 <StartDateTime>2010-10-21T01:37:51Z</StartDateTime> |
| 210 <State>COMPLETED</State> |
| 211 <EndDateTime>2010-10-21T01:39:32Z</EndDateTime> |
| 212 </ExecutionStatusDetail> |
| 213 </member> |
| 214 <member> |
| 215 <StepConfig> |
| 216 <HadoopJarStep> |
| 217 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</J
ar> |
| 218 <Args> |
| 219 <member>-mapper</member> |
| 220 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Ma
pper.py</member> |
| 221 <member>-reducer</member> |
| 222 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Re
ducer.py</member> |
| 223 <member>-input</member> |
| 224 <member>s3://example.emrtest.step1/*</member> |
| 225 <member>-output</member> |
| 226 <member>s3://example.emrtest.step3/</member> |
| 227 </Args> |
| 228 <Properties/> |
| 229 </HadoopJarStep> |
| 230 <Name>testjob_step3</Name> |
| 231 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> |
| 232 </StepConfig> |
| 233 <ExecutionStatusDetail> |
| 234 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 235 <StartDateTime>2010-10-21T01:39:32Z</StartDateTime> |
| 236 <State>COMPLETED</State> |
| 237 <EndDateTime>2010-10-21T01:41:22Z</EndDateTime> |
| 238 </ExecutionStatusDetail> |
| 239 </member> |
| 240 <member> |
| 241 <StepConfig> |
| 242 <HadoopJarStep> |
| 243 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</J
ar> |
| 244 <Args> |
| 245 <member>-mapper</member> |
| 246 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Ma
pper.py</member> |
| 247 <member>-reducer</member> |
| 248 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Re
ducer.py</member> |
| 249 <member>-input</member> |
| 250 <member>s3://example.emrtest.step1/*</member> |
| 251 <member>-output</member> |
| 252 <member>s3://example.emrtest.step4/</member> |
| 253 </Args> |
| 254 <Properties/> |
| 255 </HadoopJarStep> |
| 256 <Name>testjob_step4</Name> |
| 257 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> |
| 258 </StepConfig> |
| 259 <ExecutionStatusDetail> |
| 260 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 261 <StartDateTime>2010-10-21T01:41:22Z</StartDateTime> |
| 262 <State>COMPLETED</State> |
| 263 <EndDateTime>2010-10-21T01:43:03Z</EndDateTime> |
| 264 </ExecutionStatusDetail> |
| 265 </member> |
| 266 </Steps> |
| 267 <JobFlowId>j-3H3Q13JPFLU22</JobFlowId> |
| 268 <Instances> |
| 269 <SlaveInstanceType>m1.large</SlaveInstanceType> |
| 270 <MasterInstanceId>i-64c21609</MasterInstanceId> |
| 271 <Placement> |
| 272 <AvailabilityZone>us-east-1b</AvailabilityZone> |
| 273 </Placement> |
| 274 <InstanceGroups> |
| 275 <member> |
| 276 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 277 <InstanceRunningCount>0</InstanceRunningCount> |
| 278 <StartDateTime>2010-10-21T01:02:09Z</StartDateTime> |
| 279 <ReadyDateTime>2010-10-21T01:03:03Z</ReadyDateTime> |
| 280 <State>ENDED</State> |
| 281 <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> |
| 282 <InstanceRequestCount>1</InstanceRequestCount> |
| 283 <InstanceType>m1.large</InstanceType> |
| 284 <Market>ON_DEMAND</Market> |
| 285 <LastStateChangeReason>Job flow terminated</LastStateChangeReason> |
| 286 <InstanceRole>MASTER</InstanceRole> |
| 287 <InstanceGroupId>ig-EVMHOZJ2SCO8</InstanceGroupId> |
| 288 <Name>master</Name> |
| 289 </member> |
| 290 <member> |
| 291 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> |
| 292 <InstanceRunningCount>0</InstanceRunningCount> |
| 293 <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> |
| 294 <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime> |
| 295 <State>ENDED</State> |
| 296 <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> |
| 297 <InstanceRequestCount>9</InstanceRequestCount> |
| 298 <InstanceType>m1.large</InstanceType> |
| 299 <Market>ON_DEMAND</Market> |
| 300 <LastStateChangeReason>Job flow terminated</LastStateChangeReason> |
| 301 <InstanceRole>CORE</InstanceRole> |
| 302 <InstanceGroupId>ig-YZHDYVITVHKB</InstanceGroupId> |
| 303 <Name>slave</Name> |
| 304 </member> |
| 305 </InstanceGroups> |
| 306 <NormalizedInstanceHours>40</NormalizedInstanceHours> |
| 307 <HadoopVersion>0.20</HadoopVersion> |
| 308 <MasterInstanceType>m1.large</MasterInstanceType> |
| 309 <MasterPublicDnsName>ec2-184-72-153-139.compute-1.amazonaws.com</Maste
rPublicDnsName> |
| 310 <Ec2KeyName>myubersecurekey</Ec2KeyName> |
| 311 <InstanceCount>10</InstanceCount> |
| 312 <KeepJobFlowAliveWhenNoSteps>false</KeepJobFlowAliveWhenNoSteps> |
| 313 </Instances> |
| 314 </member> |
| 315 </JobFlows> |
| 316 </DescribeJobFlowsResult> |
| 317 <ResponseMetadata> |
| 318 <RequestId>c31e701d-dcb4-11df-b5d9-337fc7fe4773</RequestId> |
| 319 </ResponseMetadata> |
| 320 </DescribeJobFlowsResponse> |
| 321 """ |
| 322 |
| 323 |
| 324 class TestEMRResponses(unittest.TestCase): |
| 325 def _parse_xml(self, body, markers): |
| 326 rs = ResultSet(markers) |
| 327 h = handler.XmlHandler(rs, None) |
| 328 xml.sax.parseString(body, h) |
| 329 return rs |
| 330 |
| 331 def _assert_fields(self, response, **fields): |
| 332 for field, expected in fields.items(): |
| 333 actual = getattr(response, field) |
| 334 self.assertEquals(expected, actual, |
| 335 "Field %s: %r != %r" % (field, expected, actual)) |
| 336 |
| 337 def test_JobFlows_example(self): |
| 338 [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE, |
| 339 [('member', emrobject.JobFlow)]) |
| 340 self._assert_fields(jobflow, |
| 341 creationdatetime='2009-01-28T21:49:16Z', |
| 342 startdatetime='2009-01-28T21:49:16Z', |
| 343 state='STARTING', |
| 344 instancecount='4', |
| 345 jobflowid='j-3UN6WX5RRO2AG', |
| 346 loguri='mybucket/subdir/', |
| 347 name='MyJobFlowName', |
| 348 availabilityzone='us-east-1a', |
| 349 slaveinstancetype='m1.small', |
| 350 masterinstancetype='m1.small', |
| 351 ec2keyname='myec2keyname', |
| 352 keepjobflowalivewhennosteps='true') |
| 353 |
| 354 def test_JobFlows_completed(self): |
| 355 [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED, |
| 356 [('member', emrobject.JobFlow)]) |
| 357 self._assert_fields(jobflow, |
| 358 creationdatetime='2010-10-21T01:00:25Z', |
| 359 startdatetime='2010-10-21T01:03:59Z', |
| 360 enddatetime='2010-10-21T01:44:18Z', |
| 361 state='COMPLETED', |
| 362 instancecount='10', |
| 363 jobflowid='j-3H3Q13JPFLU22', |
| 364 loguri='s3n://example.emrtest.scripts/jobflow_logs/'
, |
| 365 name='RealJobFlowName', |
| 366 availabilityzone='us-east-1b', |
| 367 slaveinstancetype='m1.large', |
| 368 masterinstancetype='m1.large', |
| 369 ec2keyname='myubersecurekey', |
| 370 keepjobflowalivewhennosteps='false') |
| 371 self.assertEquals(6, len(jobflow.steps)) |
| 372 self.assertEquals(2, len(jobflow.instancegroups)) |
| 373 |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12317103:
Added gsutil to depot tools (Closed)
Base URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git@master