OLD | NEW |
| (Empty) |
1 # Portions copyright Canonical Ltd. 2009 | |
2 | |
3 import os | |
4 import sys | |
5 import StringIO | |
6 import textwrap | |
7 | |
8 from twisted.trial import unittest | |
9 from twisted.internet import defer, reactor | |
10 | |
11 from buildbot.process.base import BuildRequest | |
12 from buildbot.sourcestamp import SourceStamp | |
13 from buildbot.status.builder import SUCCESS | |
14 from buildbot.test.runutils import RunMixin | |
15 | |
16 | |
17 PENDING = 'pending' | |
18 RUNNING = 'running' | |
19 SHUTTINGDOWN = 'shutting-down' | |
20 TERMINATED = 'terminated' | |
21 | |
22 | |
23 class EC2ResponseError(Exception): | |
24 def __init__(self, code): | |
25 self.code = code | |
26 | |
27 | |
28 class Stub: | |
29 def __init__(self, **kwargs): | |
30 self.__dict__.update(kwargs) | |
31 | |
32 | |
33 class Instance: | |
34 | |
35 def __init__(self, data, ami, **kwargs): | |
36 self.data = data | |
37 self.state = PENDING | |
38 self.id = ami | |
39 self.public_dns_name = 'ec2-012-345-678-901.compute-1.amazonaws.com' | |
40 self.__dict__.update(kwargs) | |
41 self.output = Stub(name='output', output='example_output') | |
42 | |
43 def update(self): | |
44 if self.state == PENDING: | |
45 self.data.testcase.connectOneSlave(self.data.slave.slavename) | |
46 self.state = RUNNING | |
47 elif self.state == SHUTTINGDOWN: | |
48 slavename = self.data.slave.slavename | |
49 slaves = self.data.testcase.slaves | |
50 if slavename in slaves: | |
51 def discard(data): | |
52 pass | |
53 s = slaves.pop(slavename) | |
54 bot = s.getServiceNamed("bot") | |
55 for buildername in self.data.slave.slavebuilders: | |
56 remote = bot.builders[buildername].remote | |
57 if remote is None: | |
58 continue | |
59 broker = remote.broker | |
60 broker.dataReceived = discard # seal its ears | |
61 # and take away its voice | |
62 broker.transport.write = discard | |
63 # also discourage it from reconnecting once the connection | |
64 # goes away | |
65 s.bf.continueTrying = False | |
66 # stop the service for cleanliness | |
67 s.stopService() | |
68 self.state = TERMINATED | |
69 | |
70 def get_console_output(self): | |
71 return self.output | |
72 | |
73 def use_ip(self, elastic_ip): | |
74 if isinstance(elastic_ip, Stub): | |
75 elastic_ip = elastic_ip.public_ip | |
76 if self.data.addresses[elastic_ip] is not None: | |
77 raise ValueError('elastic ip already used') | |
78 self.data.addresses[elastic_ip] = self | |
79 | |
80 def stop(self): | |
81 self.state = SHUTTINGDOWN | |
82 | |
83 class Image: | |
84 | |
85 def __init__(self, data, ami, owner, location): | |
86 self.data = data | |
87 self.id = ami | |
88 self.owner = owner | |
89 self.location = location | |
90 | |
91 def run(self, **kwargs): | |
92 return Stub(name='reservation', | |
93 instances=[Instance(self.data, self.id, **kwargs)]) | |
94 | |
95 def create(klass, data, ami, owner, location): | |
96 assert ami not in data.images | |
97 self = klass(data, ami, owner, location) | |
98 data.images[ami] = self | |
99 return self | |
100 create = classmethod(create) | |
101 | |
102 | |
103 class Connection: | |
104 | |
105 def __init__(self, data): | |
106 self.data = data | |
107 | |
108 def get_all_key_pairs(self, keypair_name): | |
109 try: | |
110 return [self.data.keys[keypair_name]] | |
111 except KeyError: | |
112 raise EC2ResponseError('InvalidKeyPair.NotFound') | |
113 | |
114 def create_key_pair(self, keypair_name): | |
115 return Key.create(keypair_name, self.data.keys) | |
116 | |
117 def get_all_security_groups(self, security_name): | |
118 try: | |
119 return [self.data.security_groups[security_name]] | |
120 except KeyError: | |
121 raise EC2ResponseError('InvalidGroup.NotFound') | |
122 | |
123 def create_security_group(self, security_name, description): | |
124 assert security_name not in self.data.security_groups | |
125 res = Stub(name='security_group', value=security_name, | |
126 description=description) | |
127 self.data.security_groups[security_name] = res | |
128 return res | |
129 | |
130 def get_all_images(self, owners=None): | |
131 # return a list of images. images have .location and .id. | |
132 res = self.data.images.values() | |
133 if owners: | |
134 res = [image for image in res if image.owner in owners] | |
135 return res | |
136 | |
137 def get_image(self, machine_id): | |
138 # return image or raise an error | |
139 return self.data.images[machine_id] | |
140 | |
141 def get_all_addresses(self, elastic_ips): | |
142 res = [] | |
143 for ip in elastic_ips: | |
144 if ip in self.data.addresses: | |
145 res.append(Stub(public_ip=ip)) | |
146 else: | |
147 raise EC2ResponseError('...bad address...') | |
148 return res | |
149 | |
150 def disassociate_address(self, address): | |
151 if address not in self.data.addresses: | |
152 raise EC2ResponseError('...unknown address...') | |
153 self.data.addresses[address] = None | |
154 | |
155 | |
156 class Key: | |
157 | |
158 # this is what we would need to do if we actually needed a real key. | |
159 # We don't right now. | |
160 #def __init__(self): | |
161 # self.raw = paramiko.RSAKey.generate(256) | |
162 # f = StringIO.StringIO() | |
163 # self.raw.write_private_key(f) | |
164 # self.material = f.getvalue() | |
165 | |
166 def create(klass, name, keys): | |
167 self = klass() | |
168 self.name = name | |
169 self.keys = keys | |
170 assert name not in keys | |
171 keys[name] = self | |
172 return self | |
173 create = classmethod(create) | |
174 | |
175 def delete(self): | |
176 del self.keys[self.name] | |
177 | |
178 | |
179 class Boto: | |
180 | |
181 slave = None # must be set in setUp | |
182 | |
183 def __init__(self, testcase): | |
184 self.testcase = testcase | |
185 self.keys = {} | |
186 Key.create('latent_buildbot_slave', self.keys) | |
187 Key.create('buildbot_slave', self.keys) | |
188 kk = self.keys.keys() | |
189 kk.sort() | |
190 assert kk == ['buildbot_slave', 'latent_buildbot_slave'] | |
191 self.original_keys = dict(self.keys) | |
192 self.security_groups = { | |
193 'latent_buildbot_slave': Stub(name='security_group', | |
194 value='latent_buildbot_slave')} | |
195 self.addresses = {'127.0.0.1': None} | |
196 self.images = {} | |
197 Image.create(self, 'ami-12345', 12345667890, | |
198 'test-xx/image.manifest.xml') | |
199 Image.create(self, 'ami-AF000', 11111111111, | |
200 'test-f0a/image.manifest.xml') | |
201 Image.create(self, 'ami-CE111', 22222222222, | |
202 'test-e1b/image.manifest.xml') | |
203 Image.create(self, 'ami-ED222', 22222222222, | |
204 'test-d2c/image.manifest.xml') | |
205 Image.create(self, 'ami-FC333', 22222222222, | |
206 'test-c30d/image.manifest.xml') | |
207 Image.create(self, 'ami-DB444', 11111111111, | |
208 'test-b4e/image.manifest.xml') | |
209 Image.create(self, 'ami-BA555', 11111111111, | |
210 'test-a5f/image.manifest.xml') | |
211 | |
212 def connect_ec2(self, identifier, secret_identifier): | |
213 assert identifier == 'publickey', identifier | |
214 assert secret_identifier == 'privatekey', secret_identifier | |
215 return Connection(self) | |
216 | |
217 exception = Stub(EC2ResponseError=EC2ResponseError) | |
218 | |
219 | |
220 class Mixin(RunMixin): | |
221 | |
222 def doBuild(self): | |
223 br = BuildRequest("forced", SourceStamp(), 'test_builder') | |
224 d = br.waitUntilFinished() | |
225 self.control.getBuilder('b1').requestBuild(br) | |
226 return d | |
227 | |
228 def setUp(self): | |
229 self.boto_setUp1() | |
230 self.master.loadConfig(self.config) | |
231 self.boto_setUp2() | |
232 self.boto_setUp3() | |
233 | |
234 def boto_setUp1(self): | |
235 # debugging | |
236 #import twisted.internet.base | |
237 #twisted.internet.base.DelayedCall.debug = True | |
238 # debugging | |
239 RunMixin.setUp(self) | |
240 self.boto = boto = Boto(self) | |
241 if 'boto' not in sys.modules: | |
242 sys.modules['boto'] = boto | |
243 sys.modules['boto.exception'] = boto.exception | |
244 if 'buildbot.ec2buildslave' in sys.modules: | |
245 sys.modules['buildbot.ec2buildslave'].boto = boto | |
246 | |
247 def boto_setUp2(self): | |
248 if sys.modules['boto'] is self.boto: | |
249 del sys.modules['boto'] | |
250 del sys.modules['boto.exception'] | |
251 | |
252 def boto_setUp3(self): | |
253 self.master.startService() | |
254 self.boto.slave = self.bot1 = self.master.botmaster.slaves['bot1'] | |
255 self.bot1._poll_resolution = 0.1 | |
256 self.b1 = self.master.botmaster.builders['b1'] | |
257 | |
258 def tearDown(self): | |
259 try: | |
260 import boto | |
261 import boto.exception | |
262 except ImportError: | |
263 pass | |
264 else: | |
265 sys.modules['buildbot.ec2buildslave'].boto = boto | |
266 return RunMixin.tearDown(self) | |
267 | |
268 | |
269 class BasicConfig(Mixin, unittest.TestCase): | |
270 config = textwrap.dedent("""\ | |
271 from buildbot.process import factory | |
272 from buildbot.steps import dummy | |
273 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
274 from buildbot.config import BuilderConfig | |
275 s = factory.s | |
276 | |
277 BuildmasterConfig = c = {} | |
278 c['slaves'] = [EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large', | |
279 'ami-12345', | |
280 identifier='publickey', | |
281 secret_identifier='privatekey' | |
282 )] | |
283 c['schedulers'] = [] | |
284 c['slavePortnum'] = 0 | |
285 c['schedulers'] = [] | |
286 | |
287 f1 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)]) | |
288 | |
289 c['builders'] = [ | |
290 BuilderConfig(name='b1', slavename='bot1', factory=f1), | |
291 ] | |
292 """) | |
293 | |
294 def testSequence(self): | |
295 # test with secrets in config, a single AMI, and defaults/ | |
296 self.assertEqual(self.bot1.ami, 'ami-12345') | |
297 self.assertEqual(self.bot1.instance_type, 'm1.large') | |
298 self.assertEqual(self.bot1.keypair_name, 'latent_buildbot_slave') | |
299 self.assertEqual(self.bot1.security_name, 'latent_buildbot_slave') | |
300 # this would be appropriate if we were recreating keys. | |
301 #self.assertNotEqual(self.boto.keys['latent_buildbot_slave'], | |
302 # self.boto.original_keys['latent_buildbot_slave']) | |
303 self.failUnless(isinstance(self.bot1.get_image(), Image)) | |
304 self.assertEqual(self.bot1.get_image().id, 'ami-12345') | |
305 self.assertIdentical(self.bot1.elastic_ip, None) | |
306 self.assertIdentical(self.bot1.instance, None) | |
307 # let's start a build... | |
308 self.build_deferred = self.doBuild() | |
309 # ...and wait for the ec2 slave to show up | |
310 d = self.bot1.substantiation_deferred | |
311 d.addCallback(self._testSequence_1) | |
312 return d | |
313 def _testSequence_1(self, res): | |
314 # bot 1 is substantiated. | |
315 self.assertNotIdentical(self.bot1.slave, None) | |
316 self.failUnless(self.bot1.substantiated) | |
317 self.failUnless(isinstance(self.bot1.instance, Instance)) | |
318 self.assertEqual(self.bot1.instance.id, 'ami-12345') | |
319 self.assertEqual(self.bot1.instance.state, RUNNING) | |
320 self.assertEqual(self.bot1.instance.key_name, 'latent_buildbot_slave') | |
321 self.assertEqual(self.bot1.instance.security_groups, | |
322 ['latent_buildbot_slave']) | |
323 self.assertEqual(self.bot1.instance.instance_type, 'm1.large') | |
324 self.assertEqual(self.bot1.output.output, 'example_output') | |
325 # now we'll wait for the build to complete | |
326 d = self.build_deferred | |
327 del self.build_deferred | |
328 d.addCallback(self._testSequence_2) | |
329 return d | |
330 def _testSequence_2(self, res): | |
331 # build was a success! | |
332 self.failUnlessEqual(res.getResults(), SUCCESS) | |
333 self.failUnlessEqual(res.getSlavename(), "bot1") | |
334 # Let's let it shut down. We'll set the build_wait_timer to fire | |
335 # sooner, and wait for it to fire. | |
336 self.bot1.build_wait_timer.reset(0) | |
337 # we'll stash the instance around to look at it | |
338 self.instance = self.bot1.instance | |
339 # now we wait. | |
340 d = defer.Deferred() | |
341 reactor.callLater(0.5, d.callback, None) | |
342 d.addCallback(self._testSequence_3) | |
343 return d | |
344 def _testSequence_3(self, res): | |
345 # slave is insubstantiated | |
346 self.assertIdentical(self.bot1.slave, None) | |
347 self.failIf(self.bot1.substantiated) | |
348 self.assertIdentical(self.bot1.instance, None) | |
349 self.assertEqual(self.instance.state, TERMINATED) | |
350 del self.instance | |
351 | |
352 class ElasticIP(Mixin, unittest.TestCase): | |
353 config = textwrap.dedent("""\ | |
354 from buildbot.process import factory | |
355 from buildbot.steps import dummy | |
356 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
357 from buildbot.config import BuilderConfig | |
358 s = factory.s | |
359 | |
360 BuildmasterConfig = c = {} | |
361 c['slaves'] = [EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large', | |
362 'ami-12345', | |
363 identifier='publickey', | |
364 secret_identifier='privatekey', | |
365 elastic_ip='127.0.0.1' | |
366 )] | |
367 c['schedulers'] = [] | |
368 c['slavePortnum'] = 0 | |
369 c['schedulers'] = [] | |
370 | |
371 f1 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)]) | |
372 | |
373 c['builders'] = [ | |
374 BuilderConfig(name='b1', slavename='bot1', factory=f1), | |
375 ] | |
376 """) | |
377 | |
378 def testSequence(self): | |
379 self.assertEqual(self.bot1.elastic_ip.public_ip, '127.0.0.1') | |
380 self.assertIdentical(self.boto.addresses['127.0.0.1'], None) | |
381 # let's start a build... | |
382 d = self.doBuild() | |
383 d.addCallback(self._testSequence_1) | |
384 return d | |
385 def _testSequence_1(self, res): | |
386 # build was a success! | |
387 self.failUnlessEqual(res.getResults(), SUCCESS) | |
388 self.failUnlessEqual(res.getSlavename(), "bot1") | |
389 # we have our address | |
390 self.assertIdentical(self.boto.addresses['127.0.0.1'], | |
391 self.bot1.instance) | |
392 # Let's let it shut down. We'll set the build_wait_timer to fire | |
393 # sooner, and wait for it to fire. | |
394 self.bot1.build_wait_timer.reset(0) | |
395 d = defer.Deferred() | |
396 reactor.callLater(0.5, d.callback, None) | |
397 d.addCallback(self._testSequence_2) | |
398 return d | |
399 def _testSequence_2(self, res): | |
400 # slave is insubstantiated | |
401 self.assertIdentical(self.bot1.slave, None) | |
402 self.failIf(self.bot1.substantiated) | |
403 self.assertIdentical(self.bot1.instance, None) | |
404 # the address is free again | |
405 self.assertIdentical(self.boto.addresses['127.0.0.1'], None) | |
406 | |
407 | |
408 class Initialization(Mixin, unittest.TestCase): | |
409 | |
410 def setUp(self): | |
411 self.boto_setUp1() | |
412 | |
413 def tearDown(self): | |
414 self.boto_setUp2() | |
415 return Mixin.tearDown(self) | |
416 | |
417 def testDefaultSeparateFile(self): | |
418 # set up .ec2/aws_id | |
419 home = os.environ['HOME'] | |
420 fake_home = os.path.join(os.getcwd(), 'basedir') # see RunMixin.setUp | |
421 os.environ['HOME'] = fake_home | |
422 dir = os.path.join(fake_home, '.ec2') | |
423 os.mkdir(dir) | |
424 f = open(os.path.join(dir, 'aws_id'), 'w') | |
425 f.write('publickey\nprivatekey') | |
426 f.close() | |
427 # The Connection checks the file, so if the secret file is not parsed | |
428 # correctly, *this* is where it would fail. This is the real test. | |
429 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
430 bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large', | |
431 'ami-12345') | |
432 # for completeness, we'll show that the connection actually exists. | |
433 self.failUnless(isinstance(bot1.conn, Connection)) | |
434 # clean up. | |
435 os.environ['HOME'] = home | |
436 self.rmtree(dir) | |
437 | |
438 def testCustomSeparateFile(self): | |
439 # set up .ec2/aws_id | |
440 file_path = os.path.join(os.getcwd(), 'basedir', 'custom_aws_id') | |
441 f = open(file_path, 'w') | |
442 f.write('publickey\nprivatekey') | |
443 f.close() | |
444 # The Connection checks the file, so if the secret file is not parsed | |
445 # correctly, *this* is where it would fail. This is the real test. | |
446 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
447 bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large', | |
448 'ami-12345', aws_id_file_path=file_path) | |
449 # for completeness, we'll show that the connection actually exists. | |
450 self.failUnless(isinstance(bot1.conn, Connection)) | |
451 | |
452 def testNoAMIBroken(self): | |
453 # you must specify an AMI, or at least one of valid_ami_owners or | |
454 # valid_ami_location_regex | |
455 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
456 self.assertRaises(ValueError, EC2LatentBuildSlave, 'bot1', 'sekrit', | |
457 'm1.large', identifier='publickey', | |
458 secret_identifier='privatekey') | |
459 | |
460 def testAMIOwnerFilter(self): | |
461 # if you only specify an owner, you get the image owned by any of the | |
462 # owners that sorts last by the AMI's location. | |
463 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
464 bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large', | |
465 valid_ami_owners=[11111111111], | |
466 identifier='publickey', | |
467 secret_identifier='privatekey' | |
468 ) | |
469 self.assertEqual(bot1.get_image().location, | |
470 'test-f0a/image.manifest.xml') | |
471 bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large', | |
472 valid_ami_owners=[11111111111, | |
473 22222222222], | |
474 identifier='publickey', | |
475 secret_identifier='privatekey' | |
476 ) | |
477 self.assertEqual(bot1.get_image().location, | |
478 'test-f0a/image.manifest.xml') | |
479 bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large', | |
480 valid_ami_owners=[22222222222], | |
481 identifier='publickey', | |
482 secret_identifier='privatekey' | |
483 ) | |
484 self.assertEqual(bot1.get_image().location, | |
485 'test-e1b/image.manifest.xml') | |
486 bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large', | |
487 valid_ami_owners=12345667890, | |
488 identifier='publickey', | |
489 secret_identifier='privatekey' | |
490 ) | |
491 self.assertEqual(bot1.get_image().location, | |
492 'test-xx/image.manifest.xml') | |
493 | |
494 def testAMISimpleRegexFilter(self): | |
495 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
496 bot1 = EC2LatentBuildSlave( | |
497 'bot1', 'sekrit', 'm1.large', | |
498 valid_ami_location_regex=r'test\-[a-z]\w+/image.manifest.xml', | |
499 identifier='publickey', secret_identifier='privatekey') | |
500 self.assertEqual(bot1.get_image().location, | |
501 'test-xx/image.manifest.xml') | |
502 bot1 = EC2LatentBuildSlave( | |
503 'bot1', 'sekrit', 'm1.large', | |
504 valid_ami_location_regex=r'test\-[a-z]\d+\w/image.manifest.xml', | |
505 identifier='publickey', secret_identifier='privatekey') | |
506 self.assertEqual(bot1.get_image().location, | |
507 'test-f0a/image.manifest.xml') | |
508 bot1 = EC2LatentBuildSlave( | |
509 'bot1', 'sekrit', 'm1.large', valid_ami_owners=[22222222222], | |
510 valid_ami_location_regex=r'test\-[a-z]\d+\w/image.manifest.xml', | |
511 identifier='publickey', secret_identifier='privatekey') | |
512 self.assertEqual(bot1.get_image().location, | |
513 'test-e1b/image.manifest.xml') | |
514 | |
515 def testAMIRegexAlphaSortFilter(self): | |
516 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
517 bot1 = EC2LatentBuildSlave( | |
518 'bot1', 'sekrit', 'm1.large', | |
519 valid_ami_owners=[11111111111, 22222222222], | |
520 valid_ami_location_regex=r'test\-[a-z]\d+([a-z])/image.manifest.xml'
, | |
521 identifier='publickey', secret_identifier='privatekey') | |
522 self.assertEqual(bot1.get_image().location, | |
523 'test-a5f/image.manifest.xml') | |
524 | |
525 def testAMIRegexIntSortFilter(self): | |
526 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
527 bot1 = EC2LatentBuildSlave( | |
528 'bot1', 'sekrit', 'm1.large', | |
529 valid_ami_owners=[11111111111, 22222222222], | |
530 valid_ami_location_regex=r'test\-[a-z](\d+)[a-z]/image.manifest.xml'
, | |
531 identifier='publickey', secret_identifier='privatekey') | |
532 self.assertEqual(bot1.get_image().location, | |
533 'test-c30d/image.manifest.xml') | |
534 | |
535 def testNewSecurityGroup(self): | |
536 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
537 bot1 = EC2LatentBuildSlave( | |
538 'bot1', 'sekrit', 'm1.large', 'ami-12345', | |
539 identifier='publickey', secret_identifier='privatekey', | |
540 security_name='custom_security_name') | |
541 self.assertEqual( | |
542 self.boto.security_groups['custom_security_name'].value, | |
543 'custom_security_name') | |
544 self.assertEqual(bot1.security_name, 'custom_security_name') | |
545 | |
546 def testNewKeypairName(self): | |
547 from buildbot.ec2buildslave import EC2LatentBuildSlave | |
548 bot1 = EC2LatentBuildSlave( | |
549 'bot1', 'sekrit', 'm1.large', 'ami-12345', | |
550 identifier='publickey', secret_identifier='privatekey', | |
551 keypair_name='custom_keypair_name') | |
552 self.assertIn('custom_keypair_name', self.boto.keys) | |
553 self.assertEqual(bot1.keypair_name, 'custom_keypair_name') | |
OLD | NEW |