From 07d81d731fa63911310629d5aa9b86860238defa Mon Sep 17 00:00:00 2001 From: Pádraig Brady Date: May 16 2012 17:23:38 +0000 Subject: Updated patches from master-patches Note I manually modified the QuantumManager-..start-dnsmasq...patch so that it excludes a hunk already applied in an earlier patch. git can handle this but patch can't. I've asked git upstream for how to avoid this issue with format-patch generated patches, but pending that I've modified the patch manually to delete the Authors part. --- diff --git a/0009-Implement-quotas-for-security-groups.patch b/0009-Implement-quotas-for-security-groups.patch new file mode 100644 index 0000000..781228f --- /dev/null +++ b/0009-Implement-quotas-for-security-groups.patch @@ -0,0 +1,476 @@ +From a67db4586f70ed881d65e80035b2a25be195ce64 Mon Sep 17 00:00:00 2001 +From: Dan Prince +Date: Wed, 11 Apr 2012 21:21:02 -0400 +Subject: [PATCH] Implement quotas for security groups. + +Fixes LP Bug #969545 for Essex. + +Change-Id: I3c6a34b43f0e997b45d5e0f97faadd6720bf7752 +--- + nova/api/ec2/cloud.py | 12 +++++++ + nova/api/openstack/compute/contrib/quotas.py | 2 +- + .../openstack/compute/contrib/security_groups.py | 12 +++++++ + nova/db/api.py | 10 ++++++ + nova/db/sqlalchemy/api.py | 16 +++++++++ + nova/quota.py | 34 ++++++++++++++++++++ + nova/tests/api/ec2/test_cloud.py | 25 ++++++++++++++ + .../api/openstack/compute/contrib/test_quotas.py | 29 ++++++++++++++--- + .../compute/contrib/test_security_groups.py | 31 ++++++++++++++++++ + nova/tests/test_quota.py | 28 ++++++++++++++++ + 10 files changed, 193 insertions(+), 6 deletions(-) + +diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py +index 16df626..9e2a22e 100644 +--- a/nova/api/ec2/cloud.py ++++ b/nova/api/ec2/cloud.py +@@ -42,6 +42,7 @@ from nova.image import s3 + from nova import log as logging + from nova import network + from nova.rpc import common as rpc_common ++from nova import quota + from nova import utils + from nova import volume + +@@ -727,6 +728,13 @@ class CloudController(object): + raise exception.EC2APIError(err % values_for_rule) + postvalues.append(values_for_rule) + ++ allowed = quota.allowed_security_group_rules(context, ++ security_group['id'], ++ 1) ++ if allowed < 1: ++ msg = _("Quota exceeded, too many security group rules.") ++ raise exception.EC2APIError(msg) ++ + rule_ids = [] + for values_for_rule in postvalues: + security_group_rule = db.security_group_rule_create( +@@ -784,6 +792,10 @@ class CloudController(object): + msg = _('group %s already exists') + raise exception.EC2APIError(msg % group_name) + ++ if quota.allowed_security_groups(context, 1) < 1: ++ msg = _("Quota exceeded, too many security groups.") ++ raise exception.EC2APIError(msg) ++ + group = {'user_id': context.user_id, + 'project_id': context.project_id, + 'name': group_name, +diff --git a/nova/api/openstack/compute/contrib/quotas.py b/nova/api/openstack/compute/contrib/quotas.py +index 53e8264..6db3d92 100644 +--- a/nova/api/openstack/compute/contrib/quotas.py ++++ b/nova/api/openstack/compute/contrib/quotas.py +@@ -31,7 +31,7 @@ authorize = extensions.extension_authorizer('compute', 'quotas') + + quota_resources = ['metadata_items', 'injected_file_content_bytes', + 'volumes', 'gigabytes', 'ram', 'floating_ips', 'instances', +- 'injected_files', 'cores'] ++ 'injected_files', 'cores', 'security_groups', 'security_group_rules'] + + + class QuotaTemplate(xmlutil.TemplateBuilder): +diff --git a/nova/api/openstack/compute/contrib/security_groups.py b/nova/api/openstack/compute/contrib/security_groups.py +index 0d85c7b..281cc8c 100644 +--- a/nova/api/openstack/compute/contrib/security_groups.py ++++ b/nova/api/openstack/compute/contrib/security_groups.py +@@ -31,6 +31,7 @@ from nova import db + from nova import exception + from nova import flags + from nova import log as logging ++from nova import quota + from nova import utils + + +@@ -289,6 +290,10 @@ class SecurityGroupController(SecurityGroupControllerBase): + group_name = group_name.strip() + group_description = group_description.strip() + ++ if quota.allowed_security_groups(context, 1) < 1: ++ msg = _("Quota exceeded, too many security groups.") ++ raise exc.HTTPBadRequest(explanation=msg) ++ + LOG.audit(_("Create Security Group %s"), group_name, context=context) + self.compute_api.ensure_default_security_group(context) + if db.security_group_exists(context, context.project_id, group_name): +@@ -376,6 +381,13 @@ class SecurityGroupRulesController(SecurityGroupControllerBase): + msg = _('This rule already exists in group %s') % parent_group_id + raise exc.HTTPBadRequest(explanation=msg) + ++ allowed = quota.allowed_security_group_rules(context, ++ parent_group_id, ++ 1) ++ if allowed < 1: ++ msg = _("Quota exceeded, too many security group rules.") ++ raise exc.HTTPBadRequest(explanation=msg) ++ + security_group_rule = db.security_group_rule_create(context, values) + self.sgh.trigger_security_group_rule_create_refresh( + context, [security_group_rule['id']]) +diff --git a/nova/db/api.py b/nova/db/api.py +index b51e1e1..27f80f6 100644 +--- a/nova/db/api.py ++++ b/nova/db/api.py +@@ -1118,6 +1118,11 @@ def security_group_destroy(context, security_group_id): + return IMPL.security_group_destroy(context, security_group_id) + + ++def security_group_count_by_project(context, project_id): ++ """Count number of security groups in a project.""" ++ return IMPL.security_group_count_by_project(context, project_id) ++ ++ + #################### + + +@@ -1149,6 +1154,11 @@ def security_group_rule_get(context, security_group_rule_id): + return IMPL.security_group_rule_get(context, security_group_rule_id) + + ++def security_group_rule_count_by_group(context, security_group_id): ++ """Count rules in a given security group.""" ++ return IMPL.security_group_rule_count_by_group(context, security_group_id) ++ ++ + ################### + + +diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py +index 69e44cd..f2c3062 100644 +--- a/nova/db/sqlalchemy/api.py ++++ b/nova/db/sqlalchemy/api.py +@@ -2813,6 +2813,13 @@ def security_group_destroy(context, security_group_id): + 'updated_at': literal_column('updated_at')}) + + ++@require_context ++def security_group_count_by_project(context, project_id): ++ authorize_project_context(context, project_id) ++ return model_query(context, models.SecurityGroup, read_deleted="no").\ ++ filter_by(project_id=project_id).\ ++ count() ++ + ################### + + +@@ -2871,6 +2878,14 @@ def security_group_rule_destroy(context, security_group_rule_id): + security_group_rule.delete(session=session) + + ++@require_context ++def security_group_rule_count_by_group(context, security_group_id): ++ return model_query(context, models.SecurityGroupIngressRule, ++ read_deleted="no").\ ++ filter_by(parent_group_id=security_group_id).\ ++ count() ++ ++# + ################### + + +@@ -3018,6 +3033,7 @@ def user_update(context, user_id, values): + user_ref.save(session=session) + + ++# + ################### + + +diff --git a/nova/quota.py b/nova/quota.py +index fc49de0..12dd146 100644 +--- a/nova/quota.py ++++ b/nova/quota.py +@@ -54,6 +54,12 @@ quota_opts = [ + cfg.IntOpt('quota_max_injected_file_path_bytes', + default=255, + help='number of bytes allowed per injected file path'), ++ cfg.IntOpt('quota_security_groups', ++ default=10, ++ help='number of security groups per project'), ++ cfg.IntOpt('quota_security_group_rules', ++ default=20, ++ help='number of security rules per security group'), + ] + + FLAGS = flags.FLAGS +@@ -72,6 +78,8 @@ def _get_default_quotas(): + 'injected_files': FLAGS.quota_max_injected_files, + 'injected_file_content_bytes': + FLAGS.quota_max_injected_file_content_bytes, ++ 'security_groups': FLAGS.quota_security_groups, ++ 'security_group_rules': FLAGS.quota_security_group_rules, + } + # -1 in the quota flags means unlimited + for key in defaults.keys(): +@@ -152,6 +160,32 @@ def allowed_floating_ips(context, requested_floating_ips): + return min(requested_floating_ips, allowed_floating_ips) + + ++def allowed_security_groups(context, requested_security_groups): ++ """Check quota and return min(requested, allowed) security groups.""" ++ project_id = context.project_id ++ context = context.elevated() ++ used_sec_groups = db.security_group_count_by_project(context, project_id) ++ quota = get_project_quotas(context, project_id) ++ allowed_sec_groups = _get_request_allotment(requested_security_groups, ++ used_sec_groups, ++ quota['security_groups']) ++ return min(requested_security_groups, allowed_sec_groups) ++ ++ ++def allowed_security_group_rules(context, security_group_id, ++ requested_rules): ++ """Check quota and return min(requested, allowed) sec group rules.""" ++ project_id = context.project_id ++ context = context.elevated() ++ used_rules = db.security_group_rule_count_by_group(context, ++ security_group_id) ++ quota = get_project_quotas(context, project_id) ++ allowed_rules = _get_request_allotment(requested_rules, ++ used_rules, ++ quota['security_group_rules']) ++ return min(requested_rules, allowed_rules) ++ ++ + def _calculate_simple_quota(context, resource, requested): + """Check quota for resource; return min(requested, allowed).""" + quota = get_project_quotas(context, context.project_id) +diff --git a/nova/tests/api/ec2/test_cloud.py b/nova/tests/api/ec2/test_cloud.py +index 9ddc730..427509c 100644 +--- a/nova/tests/api/ec2/test_cloud.py ++++ b/nova/tests/api/ec2/test_cloud.py +@@ -271,6 +271,18 @@ class CloudTestCase(test.TestCase): + delete = self.cloud.delete_security_group + self.assertTrue(delete(self.context, 'testgrp')) + ++ def test_security_group_quota_limit(self): ++ self.flags(quota_security_groups=10) ++ for i in range(1, 10): ++ name = 'test name %i' % i ++ descript = 'test description %i' % i ++ create = self.cloud.create_security_group ++ result = create(self.context, name, descript) ++ ++ # 11'th group should fail ++ self.assertRaises(exception.EC2APIError, ++ create, self.context, 'foo', 'bar') ++ + def test_delete_security_group_by_id(self): + sec = db.security_group_create(self.context, + {'project_id': self.context.project_id, +@@ -436,6 +448,19 @@ class CloudTestCase(test.TestCase): + self.assertRaises(exception.EC2APIError, authz, self.context, + group_name=sec['name'], **kwargs) + ++ def test_security_group_ingress_quota_limit(self): ++ self.flags(quota_security_group_rules=20) ++ kwargs = {'project_id': self.context.project_id, 'name': 'test'} ++ sec_group = db.security_group_create(self.context, kwargs) ++ authz = self.cloud.authorize_security_group_ingress ++ for i in range(100, 120): ++ kwargs = {'to_port': i, 'from_port': i, 'ip_protocol': 'tcp'} ++ authz(self.context, group_id=sec_group['id'], **kwargs) ++ ++ kwargs = {'to_port': 121, 'from_port': 121, 'ip_protocol': 'tcp'} ++ self.assertRaises(exception.EC2APIError, authz, self.context, ++ group_id=sec_group['id'], **kwargs) ++ + def _test_authorize_security_group_no_ports_with_source_group(self, proto): + kwargs = {'project_id': self.context.project_id, 'name': 'test'} + sec = db.security_group_create(self.context, kwargs) +diff --git a/nova/tests/api/openstack/compute/contrib/test_quotas.py b/nova/tests/api/openstack/compute/contrib/test_quotas.py +index 9808717..8f7084a 100644 +--- a/nova/tests/api/openstack/compute/contrib/test_quotas.py ++++ b/nova/tests/api/openstack/compute/contrib/test_quotas.py +@@ -28,7 +28,8 @@ def quota_set(id): + return {'quota_set': {'id': id, 'metadata_items': 128, 'volumes': 10, + 'gigabytes': 1000, 'ram': 51200, 'floating_ips': 10, + 'instances': 10, 'injected_files': 5, 'cores': 20, +- 'injected_file_content_bytes': 10240}} ++ 'injected_file_content_bytes': 10240, ++ 'security_groups': 10, 'security_group_rules': 20}} + + + def quota_set_list(): +@@ -52,7 +53,10 @@ class QuotaSetsTest(test.TestCase): + 'metadata_items': 128, + 'gigabytes': 1000, + 'injected_files': 5, +- 'injected_file_content_bytes': 10240} ++ 'injected_file_content_bytes': 10240, ++ 'security_groups': 10, ++ 'security_group_rules': 20, ++ } + + quota_set = quotas.QuotaSetsController()._format_quota_set('1234', + raw_quota_set) +@@ -68,6 +72,8 @@ class QuotaSetsTest(test.TestCase): + self.assertEqual(qs['metadata_items'], 128) + self.assertEqual(qs['injected_files'], 5) + self.assertEqual(qs['injected_file_content_bytes'], 10240) ++ self.assertEqual(qs['security_groups'], 10) ++ self.assertEqual(qs['security_group_rules'], 20) + + def test_quotas_defaults(self): + uri = '/v2/fake_tenant/os-quota-sets/fake_tenant/defaults' +@@ -85,7 +91,10 @@ class QuotaSetsTest(test.TestCase): + 'floating_ips': 10, + 'metadata_items': 128, + 'injected_files': 5, +- 'injected_file_content_bytes': 10240}} ++ 'injected_file_content_bytes': 10240, ++ 'security_groups': 10, ++ 'security_group_rules': 20, ++ }} + + self.assertEqual(res_dict, expected) + +@@ -106,7 +115,9 @@ class QuotaSetsTest(test.TestCase): + 'ram': 51200, 'volumes': 10, + 'gigabytes': 1000, 'floating_ips': 10, + 'metadata_items': 128, 'injected_files': 5, +- 'injected_file_content_bytes': 10240}} ++ 'injected_file_content_bytes': 10240, ++ 'security_groups': 10, ++ 'security_group_rules': 20}} + + req = fakes.HTTPRequest.blank('/v2/fake4/os-quota-sets/update_me', + use_admin_context=True) +@@ -119,7 +130,9 @@ class QuotaSetsTest(test.TestCase): + 'ram': 51200, 'volumes': 10, + 'gigabytes': 1000, 'floating_ips': 10, + 'metadata_items': 128, 'injected_files': 5, +- 'injected_file_content_bytes': 10240}} ++ 'injected_file_content_bytes': 10240, ++ 'security_groups': 10, ++ 'security_group_rules': 20}} + + req = fakes.HTTPRequest.blank('/v2/fake4/os-quota-sets/update_me') + self.assertRaises(webob.exc.HTTPForbidden, self.controller.update, +@@ -143,6 +156,8 @@ class QuotaXMLSerializerTest(test.TestCase): + floating_ips=60, + instances=70, + injected_files=80, ++ security_groups=10, ++ security_group_rules=20, + cores=90)) + text = self.serializer.serialize(exemplar) + +@@ -166,6 +181,8 @@ class QuotaXMLSerializerTest(test.TestCase): + floating_ips='60', + instances='70', + injected_files='80', ++ security_groups='10', ++ security_group_rules='20', + cores='90')) + intext = ("\n" + '' +@@ -178,6 +195,8 @@ class QuotaXMLSerializerTest(test.TestCase): + '60' + '70' + '80' ++ '10' ++ '20' + '90' + '') + +diff --git a/nova/tests/api/openstack/compute/contrib/test_security_groups.py b/nova/tests/api/openstack/compute/contrib/test_security_groups.py +index 0cf66ec..8cc4cc6 100644 +--- a/nova/tests/api/openstack/compute/contrib/test_security_groups.py ++++ b/nova/tests/api/openstack/compute/contrib/test_security_groups.py +@@ -25,12 +25,15 @@ from nova.api.openstack.compute.contrib import security_groups + from nova.api.openstack import wsgi + import nova.db + from nova import exception ++from nova import flags + from nova import test + from nova.tests.api.openstack import fakes + + + FAKE_UUID = 'a47ae74e-ab08-447f-8eee-ffd43fc46c16' + ++FLAGS = flags.FLAGS ++ + + class AttrDict(dict): + def __getattr__(self, k): +@@ -219,6 +222,18 @@ class TestSecurityGroups(test.TestCase): + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.create, + req, {'security_group': sg}) + ++ def test_create_security_group_quota_limit(self): ++ req = fakes.HTTPRequest.blank('/v2/fake/os-security-groups') ++ for num in range(1, FLAGS.quota_security_groups): ++ name = 'test%s' % num ++ sg = security_group_template(name=name) ++ res_dict = self.controller.create(req, {'security_group': sg}) ++ self.assertEqual(res_dict['security_group']['name'], name) ++ ++ sg = security_group_template() ++ self.assertRaises(webob.exc.HTTPBadRequest, self.controller.create, ++ req, {'security_group': sg}) ++ + def test_get_security_group_list(self): + groups = [] + for i, name in enumerate(['default', 'test']): +@@ -894,6 +909,22 @@ class TestSecurityGroupRules(test.TestCase): + self.assertRaises(webob.exc.HTTPNotFound, self.controller.delete, + req, '22222222222222') + ++ def test_create_rule_quota_limit(self): ++ req = fakes.HTTPRequest.blank('/v2/fake/os-security-group-rules') ++ for num in range(100, 100 + FLAGS.quota_security_group_rules): ++ rule = { ++ 'ip_protocol': 'tcp', 'from_port': num, ++ 'to_port': num, 'parent_group_id': '2', 'group_id': '1' ++ } ++ self.controller.create(req, {'security_group_rule': rule}) ++ ++ rule = { ++ 'ip_protocol': 'tcp', 'from_port': '121', 'to_port': '121', ++ 'parent_group_id': '2', 'group_id': '1' ++ } ++ self.assertRaises(webob.exc.HTTPBadRequest, self.controller.create, ++ req, {'security_group_rule': rule}) ++ + + class TestSecurityGroupRulesXMLDeserializer(unittest.TestCase): + +diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py +index 28c92ca..8cc5577 100644 +--- a/nova/tests/test_quota.py ++++ b/nova/tests/test_quota.py +@@ -235,6 +235,34 @@ class QuotaTestCase(test.TestCase): + floating_ips = quota.allowed_floating_ips(self.context, 101) + self.assertEqual(floating_ips, 101) + ++ def test_unlimited_security_groups(self): ++ self.flags(quota_security_groups=10) ++ security_groups = quota.allowed_security_groups(self.context, 100) ++ self.assertEqual(security_groups, 10) ++ db.quota_create(self.context, self.project_id, 'security_groups', None) ++ security_groups = quota.allowed_security_groups(self.context, 100) ++ self.assertEqual(security_groups, 100) ++ security_groups = quota.allowed_security_groups(self.context, 101) ++ self.assertEqual(security_groups, 101) ++ ++ def test_unlimited_security_group_rules(self): ++ ++ def fake_security_group_rule_count_by_group(context, sec_group_id): ++ return 0 ++ ++ self.stubs.Set(db, 'security_group_rule_count_by_group', ++ fake_security_group_rule_count_by_group) ++ ++ self.flags(quota_security_group_rules=20) ++ rules = quota.allowed_security_group_rules(self.context, 1234, 100) ++ self.assertEqual(rules, 20) ++ db.quota_create(self.context, self.project_id, 'security_group_rules', ++ None) ++ rules = quota.allowed_security_group_rules(self.context, 1234, 100) ++ self.assertEqual(rules, 100) ++ rules = quota.allowed_security_group_rules(self.context, 1234, 101) ++ self.assertEqual(rules, 101) ++ + def test_unlimited_metadata_items(self): + self.flags(quota_metadata_items=10) + items = quota.allowed_metadata_items(self.context, 100) diff --git a/0009-ensure-atomic-manipulation-of-libvirt-disk-images.patch b/0009-ensure-atomic-manipulation-of-libvirt-disk-images.patch deleted file mode 100644 index ed791d7..0000000 --- a/0009-ensure-atomic-manipulation-of-libvirt-disk-images.patch +++ /dev/null @@ -1,218 +0,0 @@ -From 5f41789556eb850732ebaa2fe3f175209ce2c198 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?P=C3=A1draig=20Brady?= -Date: Fri, 16 Mar 2012 03:43:49 +0000 -Subject: [PATCH] ensure atomic manipulation of libvirt disk images - -This pattern could probably be used elsewhere, -but only libvirt disk images are considered for now. -This change ensures there are no stale files left -anywhere in the path from glance, through the libvirt image cache. -These could cause subsequent operational errors either -directly or indirectly through disk wastage. - -* nova/utils.py: Add a new remove_path_on_error() context manager -that is used to remove the passed PATH on a raised exception. -* nova/virt/images.py: Ensure temporary downloaded and -converted images are protected. -* nova/virt/libvirt/connection.py: Ensure all the images in -the image cache and instance dirs are protected. - -Change-Id: I81a5407665a6998128c0dee41387ef00ebddeb4d ---- - nova/utils.py | 21 +++++++++-- - nova/virt/images.py | 69 +++++++++++++++++---------------------- - nova/virt/libvirt/connection.py | 16 ++++++--- - 3 files changed, 57 insertions(+), 49 deletions(-) - -diff --git a/nova/utils.py b/nova/utils.py -index 819929a..7188d98 100644 ---- a/nova/utils.py -+++ b/nova/utils.py -@@ -21,6 +21,7 @@ - - import contextlib - import datetime -+import errno - import functools - import hashlib - import inspect -@@ -1013,8 +1014,8 @@ def cleanup_file_locks(): - continue - try: - stat_info = os.stat(os.path.join(FLAGS.lock_path, filename)) -- except OSError as (errno, strerror): -- if errno == 2: # doesn't exist -+ except OSError as e: -+ if e.errno == errno.ENOENT: - continue - else: - raise -@@ -1033,8 +1034,8 @@ def delete_if_exists(pathname): - - try: - os.unlink(pathname) -- except OSError as (errno, strerror): -- if errno == 2: # doesn't exist -+ except OSError as e: -+ if e.errno == errno.ENOENT: - return - else: - raise -@@ -1344,6 +1345,18 @@ def logging_error(message): - LOG.exception(message) - - -+@contextlib.contextmanager -+def remove_path_on_error(path): -+ """Protect code that wants to operate on PATH atomically. -+ Any exception will cause PATH to be removed. -+ """ -+ try: -+ yield -+ except Exception: -+ with save_and_reraise_exception(): -+ delete_if_exists(path) -+ -+ - def make_dev_path(dev, partition=None, base='/dev'): - """Return a path to a particular device. - -diff --git a/nova/virt/images.py b/nova/virt/images.py -index 1e0ae0a..626f3ff 100644 ---- a/nova/virt/images.py -+++ b/nova/virt/images.py -@@ -51,18 +51,10 @@ def fetch(context, image_href, path, _user_id, _project_id): - # checked before we got here. - (image_service, image_id) = nova.image.get_image_service(context, - image_href) -- try: -+ with utils.remove_path_on_error(path): - with open(path, "wb") as image_file: - metadata = image_service.get(context, image_id, image_file) -- except Exception: -- with utils.save_and_reraise_exception(): -- try: -- os.unlink(path) -- except OSError, e: -- if e.errno != errno.ENOENT: -- LOG.warn("unable to remove stale image '%s': %s" % -- (path, e.strerror)) -- return metadata -+ return metadata - - - def fetch_to_raw(context, image_href, path, user_id, project_id): -@@ -85,37 +77,36 @@ def fetch_to_raw(context, image_href, path, user_id, project_id): - - return(data) - -- data = _qemu_img_info(path_tmp) -- -- fmt = data.get("file format") -- if fmt is None: -- os.unlink(path_tmp) -- raise exception.ImageUnacceptable( -- reason=_("'qemu-img info' parsing failed."), image_id=image_href) -- -- if "backing file" in data: -- backing_file = data['backing file'] -- os.unlink(path_tmp) -- raise exception.ImageUnacceptable(image_id=image_href, -- reason=_("fmt=%(fmt)s backed by: %(backing_file)s") % locals()) -- -- if fmt != "raw" and FLAGS.force_raw_images: -- staged = "%s.converted" % path -- LOG.debug("%s was %s, converting to raw" % (image_href, fmt)) -- out, err = utils.execute('qemu-img', 'convert', '-O', 'raw', -- path_tmp, staged) -- os.unlink(path_tmp) -- -- data = _qemu_img_info(staged) -- if data.get('file format', None) != "raw": -- os.unlink(staged) -+ with utils.remove_path_on_error(path_tmp): -+ data = _qemu_img_info(path_tmp) -+ -+ fmt = data.get("file format") -+ if fmt is None: -+ raise exception.ImageUnacceptable( -+ reason=_("'qemu-img info' parsing failed."), -+ image_id=image_href) -+ -+ if "backing file" in data: -+ backing_file = data['backing file'] - raise exception.ImageUnacceptable(image_id=image_href, -- reason=_("Converted to raw, but format is now %s") % -- data.get('file format', None)) -+ reason=_("fmt=%(fmt)s backed by: %(backing_file)s") % locals()) -+ -+ if fmt != "raw" and FLAGS.force_raw_images: -+ staged = "%s.converted" % path -+ LOG.debug("%s was %s, converting to raw" % (image_href, fmt)) -+ with utils.remove_path_on_error(staged): -+ out, err = utils.execute('qemu-img', 'convert', '-O', 'raw', -+ path_tmp, staged) -+ -+ data = _qemu_img_info(staged) -+ if data.get('file format', None) != "raw": -+ raise exception.ImageUnacceptable(image_id=image_href, -+ reason=_("Converted to raw, but format is now %s") % -+ data.get('file format', None)) - -- os.rename(staged, path) -+ os.rename(staged, path) - -- else: -- os.rename(path_tmp, path) -+ else: -+ os.rename(path_tmp, path) - - return metadata -diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py -index 888be92..2ade19a 100644 ---- a/nova/virt/libvirt/connection.py -+++ b/nova/virt/libvirt/connection.py -@@ -1094,7 +1094,8 @@ class LibvirtConnection(driver.ComputeDriver): - @utils.synchronized(fname) - def call_if_not_exists(base, fn, *args, **kwargs): - if not os.path.exists(base): -- fn(target=base, *args, **kwargs) -+ with utils.remove_path_on_error(base): -+ fn(target=base, *args, **kwargs) - - if cow or not generating: - call_if_not_exists(base, fn, *args, **kwargs) -@@ -1110,8 +1111,9 @@ class LibvirtConnection(driver.ComputeDriver): - size_gb = size / (1024 * 1024 * 1024) - cow_base += "_%d" % size_gb - if not os.path.exists(cow_base): -- libvirt_utils.copy_image(base, cow_base) -- disk.extend(cow_base, size) -+ with utils.remove_path_on_error(cow_base): -+ libvirt_utils.copy_image(base, cow_base) -+ disk.extend(cow_base, size) - libvirt_utils.create_cow_image(cow_base, target) - elif not generating: - libvirt_utils.copy_image(base, target) -@@ -1121,7 +1123,8 @@ class LibvirtConnection(driver.ComputeDriver): - if size: - disk.extend(target, size) - -- copy_and_extend(cow, generating, base, target, size) -+ with utils.remove_path_on_error(target): -+ copy_and_extend(cow, generating, base, target, size) - - @staticmethod - def _create_local(target, local_size, unit='G', -@@ -1291,8 +1294,9 @@ class LibvirtConnection(driver.ComputeDriver): - project_id=instance['project_id'],) - elif config_drive: - label = 'config' -- self._create_local(basepath('disk.config'), 64, unit='M', -- fs_format='msdos', label=label) # 64MB -+ with utils.remove_path_on_error(basepath('disk.config')): -+ self._create_local(basepath('disk.config'), 64, unit='M', -+ fs_format='msdos', label=label) # 64MB - - if instance['key_data']: - key = str(instance['key_data']) diff --git a/0010-Delete-fixed_ips-when-network-is-deleted.patch b/0010-Delete-fixed_ips-when-network-is-deleted.patch new file mode 100644 index 0000000..03ce059 --- /dev/null +++ b/0010-Delete-fixed_ips-when-network-is-deleted.patch @@ -0,0 +1,87 @@ +From 015744e92e601036ddcd77bd2fbed966172cb759 Mon Sep 17 00:00:00 2001 +From: Vishvananda Ishaya +Date: Tue, 3 Apr 2012 11:30:57 -0700 +Subject: [PATCH] Delete fixed_ips when network is deleted + + * adds failing test + * adds exception that is raised when network is in use + * fixes bug 754900 + +Change-Id: Ib95dc5927561b979b1eea237d4d6dc323483d4a5 +--- + nova/db/sqlalchemy/api.py | 13 +++++++++++++ + nova/exception.py | 4 ++++ + nova/tests/test_db_api.py | 19 +++++++++++++++++++ + 3 files changed, 36 insertions(+), 0 deletions(-) + +diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py +index f2c3062..03ac987 100644 +--- a/nova/db/sqlalchemy/api.py ++++ b/nova/db/sqlalchemy/api.py +@@ -1898,8 +1898,21 @@ def network_create_safe(context, values): + def network_delete_safe(context, network_id): + session = get_session() + with session.begin(): ++ result = session.query(models.FixedIp).\ ++ filter_by(network_id=network_id).\ ++ filter_by(deleted=False).\ ++ filter_by(allocated=True).\ ++ all() ++ if result: ++ raise exception.NetworkInUse(network_id=network_id) + network_ref = network_get(context, network_id=network_id, + session=session) ++ session.query(models.FixedIp).\ ++ filter_by(network_id=network_id).\ ++ filter_by(deleted=False).\ ++ update({'deleted': True, ++ 'updated_at': literal_column('updated_at'), ++ 'deleted_at': utils.utcnow()}) + session.delete(network_ref) + + +diff --git a/nova/exception.py b/nova/exception.py +index eb0bf38..da067b6 100644 +--- a/nova/exception.py ++++ b/nova/exception.py +@@ -525,6 +525,10 @@ class StorageRepositoryNotFound(NotFound): + message = _("Cannot find SR to read/write VDI.") + + ++class NetworkInUse(NovaException): ++ message = _("Network %(network_id)s is still in use.") ++ ++ + class NetworkNotCreated(NovaException): + message = _("%(req)s is required to create a network.") + +diff --git a/nova/tests/test_db_api.py b/nova/tests/test_db_api.py +index 8b73580..28f3558 100644 +--- a/nova/tests/test_db_api.py ++++ b/nova/tests/test_db_api.py +@@ -136,6 +136,25 @@ class DbApiTestCase(test.TestCase): + db_network = db.network_get(ctxt, network.id) + self.assertEqual(network.uuid, db_network.uuid) + ++ def test_network_delete_safe(self): ++ ctxt = context.get_admin_context() ++ values = {'host': 'localhost', 'project_id': 'project1'} ++ network = db.network_create_safe(ctxt, values) ++ db_network = db.network_get(ctxt, network.id) ++ values = {'network_id': network['id'], 'address': 'fake1'} ++ address1 = db.fixed_ip_create(ctxt, values) ++ values = {'network_id': network['id'], ++ 'address': 'fake2', ++ 'allocated': True} ++ address2 = db.fixed_ip_create(ctxt, values) ++ self.assertRaises(exception.NetworkInUse, ++ db.network_delete_safe, ctxt, network['id']) ++ db.fixed_ip_update(ctxt, address2, {'allocated': False}) ++ network = db.network_delete_safe(ctxt, network['id']) ++ ctxt = ctxt.elevated(read_deleted='yes') ++ fixed_ip = db.fixed_ip_get_by_address(ctxt, address1) ++ self.assertTrue(fixed_ip['deleted']) ++ + def test_network_create_with_duplicate_vlan(self): + ctxt = context.get_admin_context() + values1 = {'host': 'localhost', 'project_id': 'project1', 'vlan': 1} diff --git a/0010-Ensure-we-don-t-access-the-net-when-building-docs.patch b/0010-Ensure-we-don-t-access-the-net-when-building-docs.patch deleted file mode 100644 index df40317..0000000 --- a/0010-Ensure-we-don-t-access-the-net-when-building-docs.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 3057a7b6d56a6ad26d70af516fd28ee55938eb64 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?P=C3=A1draig=20Brady?= -Date: Fri, 6 Jan 2012 12:16:34 +0000 -Subject: [PATCH] Ensure we don't access the net when building docs - -(Note, this has not been sent upstream) - -Change-Id: I9d02fb4053a8106672aded1614a2850e21603eb2 ---- - doc/source/conf.py | 2 +- - 1 files changed, 1 insertions(+), 1 deletions(-) - -diff --git a/doc/source/conf.py b/doc/source/conf.py -index 8ced294..7df59cd 100644 ---- a/doc/source/conf.py -+++ b/doc/source/conf.py -@@ -25,7 +25,7 @@ sys.path.insert(0, os.path.abspath('./')) - # Add any Sphinx extension module names here, as strings. They can be extensions - # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. - --extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'ext.nova_todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig','sphinx.ext.graphviz'] -+extensions = ['sphinx.ext.autodoc', 'ext.nova_todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig','sphinx.ext.graphviz'] - - # autodoc generation is a bit aggressive and a nuisance when doing heavy text edit cycles. - # execute "export SPHINX_DEBUG=1" in your terminal to disable diff --git a/0011-Xen-Pass-session-to-destroy_vdi.patch b/0011-Xen-Pass-session-to-destroy_vdi.patch new file mode 100644 index 0000000..2212baa --- /dev/null +++ b/0011-Xen-Pass-session-to-destroy_vdi.patch @@ -0,0 +1,25 @@ +From 6c68ef55d966e6c8a2591886535a1590a6da72ad Mon Sep 17 00:00:00 2001 +From: Renuka Apte +Date: Wed, 25 Apr 2012 15:55:14 -0700 +Subject: [PATCH] Xen: Pass session to destroy_vdi + +fixes bug 988615 + +Change-Id: I34c59ff536abfdff9221cdb3d9ecc45d1e7a1a90 +--- + nova/virt/xenapi/volumeops.py | 2 +- + 1 files changed, 1 insertions(+), 1 deletions(-) + +diff --git a/nova/virt/xenapi/volumeops.py b/nova/virt/xenapi/volumeops.py +index 2f3aafb..8333c08 100644 +--- a/nova/virt/xenapi/volumeops.py ++++ b/nova/virt/xenapi/volumeops.py +@@ -63,7 +63,7 @@ class VolumeOps(object): + if vdi_ref is None: + raise exception.Error(_('Could not find VDI ref')) + +- vm_utils.VMHelper.destroy_vdi(vdi_ref) ++ vm_utils.VMHelper.destroy_vdi(self._session, vdi_ref) + + def create_sr(self, label, params): + LOG.debug(_("Creating SR %s") % label) diff --git a/0011-fix-useexisting-deprecation-warnings.patch b/0011-fix-useexisting-deprecation-warnings.patch deleted file mode 100644 index 856cd7a..0000000 --- a/0011-fix-useexisting-deprecation-warnings.patch +++ /dev/null @@ -1,49 +0,0 @@ -From a2760a769da807184deb0f45ea6ec5f84afd5251 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?P=C3=A1draig=20Brady?= -Date: Thu, 8 Mar 2012 16:32:30 +0000 -Subject: [PATCH] fix useexisting deprecation warnings - -Fixes deprecation warnings when using sqlalchemy >= 0.7.0 -Fixes bug 941951 - -Change-Id: Iaa57153f99c60c67a14c1dca849188937bdc5dee ---- - .../075_convert_bw_usage_to_store_network_id.py | 4 ++-- - .../versions/081_drop_instance_id_bw_cache.py | 2 +- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/nova/db/sqlalchemy/migrate_repo/versions/075_convert_bw_usage_to_store_network_id.py b/nova/db/sqlalchemy/migrate_repo/versions/075_convert_bw_usage_to_store_network_id.py -index b275524..4ff3d99 100644 ---- a/nova/db/sqlalchemy/migrate_repo/versions/075_convert_bw_usage_to_store_network_id.py -+++ b/nova/db/sqlalchemy/migrate_repo/versions/075_convert_bw_usage_to_store_network_id.py -@@ -46,7 +46,7 @@ def upgrade(migrate_engine): - Column('last_refreshed', DateTime(timezone=False)), - Column('bw_in', BigInteger()), - Column('bw_out', BigInteger()), -- useexisting=True) -+ extend_existing=True) - mac_column = Column('mac', String(255)) - bw_usage_cache.create_column(mac_column) - -@@ -81,7 +81,7 @@ def downgrade(migrate_engine): - Column('last_refreshed', DateTime(timezone=False)), - Column('bw_in', BigInteger()), - Column('bw_out', BigInteger()), -- useexisting=True) -+ extend_existing=True) - - network_label_column = Column('network_label', String(255)) - bw_usage_cache.create_column(network_label_column) -diff --git a/nova/db/sqlalchemy/migrate_repo/versions/081_drop_instance_id_bw_cache.py b/nova/db/sqlalchemy/migrate_repo/versions/081_drop_instance_id_bw_cache.py -index c6687ac..a607ed3 100644 ---- a/nova/db/sqlalchemy/migrate_repo/versions/081_drop_instance_id_bw_cache.py -+++ b/nova/db/sqlalchemy/migrate_repo/versions/081_drop_instance_id_bw_cache.py -@@ -37,7 +37,7 @@ def upgrade(migrate_engine): - Column('last_refreshed', DateTime(timezone=False)), - Column('bw_in', BigInteger()), - Column('bw_out', BigInteger()), -- useexisting=True) -+ extend_existing=True) - - bw_usage_cache.drop_column('instance_id') - diff --git a/0012-add-libvirt_inject_key-flag.patch b/0012-add-libvirt_inject_key-flag.patch new file mode 100644 index 0000000..81f726a --- /dev/null +++ b/0012-add-libvirt_inject_key-flag.patch @@ -0,0 +1,33 @@ +From 5ab505191c3600fc4f4b7b128a04f5c9c8f74bc1 Mon Sep 17 00:00:00 2001 +From: Peng Yong +Date: Mon, 2 Apr 2012 23:36:20 +0800 +Subject: [PATCH] add libvirt_inject_key flag fix bug #971640 + +Change-Id: I48efc5babdd9b233342a33c87c461aabf5f5915b +--- + nova/virt/libvirt/connection.py | 5 ++++- + 1 files changed, 4 insertions(+), 1 deletions(-) + +diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py +index 888be92..1a00db6 100644 +--- a/nova/virt/libvirt/connection.py ++++ b/nova/virt/libvirt/connection.py +@@ -105,6 +105,9 @@ libvirt_opts = [ + default=False, + help='Inject the admin password at boot time, ' + 'without an agent.'), ++ cfg.BoolOpt('libvirt_inject_key', ++ default=True, ++ help='Inject the ssh public key at boot time'), + cfg.BoolOpt('use_usb_tablet', + default=True, + help='Sync virtual and real mouse cursors in Windows VMs'), +@@ -1294,7 +1297,7 @@ class LibvirtConnection(driver.ComputeDriver): + self._create_local(basepath('disk.config'), 64, unit='M', + fs_format='msdos', label=label) # 64MB + +- if instance['key_data']: ++ if FLAGS.libvirt_inject_key and instance['key_data']: + key = str(instance['key_data']) + else: + key = None diff --git a/0012-support-a-configurable-libvirt-injection-partition.patch b/0012-support-a-configurable-libvirt-injection-partition.patch deleted file mode 100644 index deeb2f4..0000000 --- a/0012-support-a-configurable-libvirt-injection-partition.patch +++ /dev/null @@ -1,80 +0,0 @@ -From fe56b346bb01559248a1ea3c59c2a4baf95f3646 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?P=C3=A1draig=20Brady?= -Date: Wed, 18 Apr 2012 23:27:31 +0100 -Subject: [PATCH] support a configurable libvirt injection partition - -This is useful if all guest images have the same structure, -and the root partition is not the first partition. - -This is also handy to enable inspection in libguestfs, -which can handle disparate and complicated image layouts. - -In future we may change to a StrOpt to support -searching by partition label. - -Change-Id: Ie94d61bec8fe4b41d6d2d6d3efa9a4364cf027fe - -Conflicts: - - nova/virt/libvirt/connection.py ---- - nova/virt/disk/mount.py | 6 ++++-- - nova/virt/libvirt/connection.py | 12 ++++++++---- - 2 files changed, 12 insertions(+), 6 deletions(-) - -diff --git a/nova/virt/disk/mount.py b/nova/virt/disk/mount.py -index 4fb5dda..11959b2 100644 ---- a/nova/virt/disk/mount.py -+++ b/nova/virt/disk/mount.py -@@ -58,7 +58,9 @@ class Mount(object): - """Map partitions of the device to the file system namespace.""" - assert(os.path.exists(self.device)) - -- if self.partition: -+ if self.partition == -1: -+ self.error = _('partition search unsupported with %s') % self.mode -+ elif self.partition: - map_path = '/dev/mapper/%sp%s' % (os.path.basename(self.device), - self.partition) - assert(not os.path.exists(map_path)) -@@ -73,7 +75,7 @@ class Mount(object): - # so given we only use it when we expect a partitioned image, fail - if not os.path.exists(map_path): - if not err: -- err = _('no partitions found') -+ err = _('partition %s not found') % self.partition - self.error = _('Failed to map partitions: %s') % err - else: - self.mapped_device = map_path -diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py -index 2ade19a..5a9db4e 100644 ---- a/nova/virt/libvirt/connection.py -+++ b/nova/virt/libvirt/connection.py -@@ -105,6 +105,11 @@ libvirt_opts = [ - default=False, - help='Inject the admin password at boot time, ' - 'without an agent.'), -+ cfg.IntOpt('libvirt_inject_partition', -+ default=1, -+ help='The partition to inject to : ' -+ '-1 => inspect (libguestfs only), 0 => not partitioned, ' -+ '>0 => partition number'), - cfg.BoolOpt('use_usb_tablet', - default=True, - help='Sync virtual and real mouse cursors in Windows VMs'), -@@ -1271,12 +1276,11 @@ class LibvirtConnection(driver.ComputeDriver): - cow=FLAGS.use_cow_images, - swap_mb=swap_mb) - -- # For now, we assume that if we're not using a kernel, we're using a -- # partitioned disk image where the target partition is the first -- # partition - target_partition = None - if not instance['kernel_id']: -- target_partition = "1" -+ target_partition = FLAGS.libvirt_inject_partition -+ if target_partition == 0: -+ target_partition = None - - config_drive_id = instance.get('config_drive_id') - config_drive = instance.get('config_drive') diff --git a/0013-Cloudpipe-tap-vpn-not-always-working.patch b/0013-Cloudpipe-tap-vpn-not-always-working.patch new file mode 100644 index 0000000..0ce7544 --- /dev/null +++ b/0013-Cloudpipe-tap-vpn-not-always-working.patch @@ -0,0 +1,51 @@ +From 7c64de95f422add711bcdf5821310435e7be0199 Mon Sep 17 00:00:00 2001 +From: Cor Cornelisse +Date: Fri, 6 Apr 2012 15:54:16 +0200 +Subject: [PATCH] Cloudpipe tap vpn not always working + +Fixes bug 975043 + +Since Essex, all instances will have an eth0 MAC address in the range +of FA:16:3E, which is near the end of the MAC address space. + +When openvpn is started, a TAP interface is created with a random +generated MAC address. Chances are high the generated MAC address is +lower in value than the eth0 MAC address. Once the tap interface is +added to the bridge interface, the bridge interface will no longer have +the eth0 MAC address, but take over the TAP MAC address. This is a +feature of the linux kernel, whereby a bridge interface will take the +MAC address with the lowest value amongst its interfaces. After the ARP +entries expire, this will result in the cloudpipe instance being no +longer reachable. + +This fix, randomly generates a MAC address starting with FA:17:3E, which +is greater than FA, and will thus ensure the brige will keep the eth0 MAC +address. + +Change-Id: I0bd994b6dc7a92738ed23cd62ee42a021fd394e2 +--- + nova/cloudpipe/bootscript.template | 5 +++++ + 1 files changed, 5 insertions(+), 0 deletions(-) + +diff --git a/nova/cloudpipe/bootscript.template b/nova/cloudpipe/bootscript.template +index 94dea3f..0fe38b7 100755 +--- a/nova/cloudpipe/bootscript.template ++++ b/nova/cloudpipe/bootscript.template +@@ -24,6 +24,10 @@ export VPN_IP=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 + export BROADCAST=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f3 | awk '{print $$1}'` + export DHCP_MASK=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f4 | awk '{print $$1}'` + export GATEWAY=`netstat -r | grep default | cut -d' ' -f10` ++# Need a higher valued MAC address than eth0, to prevent the TAP MAC address ++# from becoming the bridge MAC address. Since Essex eth0 MAC starts with ++# FA:16:3E, we'll thus generate a MAC starting with FA:17:3E to be higher than eth0. ++export RANDOM_TAP_MAC=`openssl rand -hex 8 | sed 's/\(..\)/\1:/g' | cut -b-8 | awk '{print "FA:17:3E:"$$1}'` + + DHCP_LOWER=`echo $$BROADCAST | awk -F. '{print $$1"."$$2"."$$3"." $$4 - ${num_vpn} }'` + DHCP_UPPER=`echo $$BROADCAST | awk -F. '{print $$1"."$$2"."$$3"." $$4 - 1 }'` +@@ -47,5 +51,6 @@ sed -i -e s/max-clients\ 1/max-clients\ 10/g server.conf + echo "push \"route ${dmz_net} ${dmz_mask} $$GATEWAY\"" >> server.conf + echo "duplicate-cn" >> server.conf + echo "crl-verify /etc/openvpn/crl.pem" >> server.conf ++echo "lladdr $$RANDOM_TAP_MAC" >> server.conf + + /etc/init.d/openvpn start diff --git a/0013-enforce-quota-on-security-group-rules.patch b/0013-enforce-quota-on-security-group-rules.patch deleted file mode 100644 index 23c1173..0000000 --- a/0013-enforce-quota-on-security-group-rules.patch +++ /dev/null @@ -1,477 +0,0 @@ -From 8ed501ea6d75228b3490a7c0176cd2f7501d0180 Mon Sep 17 00:00:00 2001 -From: Dan Prince -Date: Thu, 19 Apr 2012 14:57:16 +0100 -Subject: [PATCH] enforce quota on security group rules - -There is no limit on the number of security group rules a user can create. -By creating a very large set of rules, an unreasonable number of -iptables rules will be created on compute nodes, resulting in a denial -of service. ---- - nova/api/ec2/cloud.py | 12 +++++++ - nova/api/openstack/compute/contrib/quotas.py | 2 +- - .../openstack/compute/contrib/security_groups.py | 12 +++++++ - nova/db/api.py | 10 ++++++ - nova/db/sqlalchemy/api.py | 16 +++++++++ - nova/quota.py | 34 ++++++++++++++++++++ - nova/tests/api/ec2/test_cloud.py | 25 ++++++++++++++ - .../api/openstack/compute/contrib/test_quotas.py | 29 ++++++++++++++--- - .../compute/contrib/test_security_groups.py | 31 ++++++++++++++++++ - nova/tests/test_quota.py | 28 ++++++++++++++++ - 10 files changed, 193 insertions(+), 6 deletions(-) - -diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py -index 16df626..9e2a22e 100644 ---- a/nova/api/ec2/cloud.py -+++ b/nova/api/ec2/cloud.py -@@ -42,6 +42,7 @@ from nova.image import s3 - from nova import log as logging - from nova import network - from nova.rpc import common as rpc_common -+from nova import quota - from nova import utils - from nova import volume - -@@ -727,6 +728,13 @@ class CloudController(object): - raise exception.EC2APIError(err % values_for_rule) - postvalues.append(values_for_rule) - -+ allowed = quota.allowed_security_group_rules(context, -+ security_group['id'], -+ 1) -+ if allowed < 1: -+ msg = _("Quota exceeded, too many security group rules.") -+ raise exception.EC2APIError(msg) -+ - rule_ids = [] - for values_for_rule in postvalues: - security_group_rule = db.security_group_rule_create( -@@ -784,6 +792,10 @@ class CloudController(object): - msg = _('group %s already exists') - raise exception.EC2APIError(msg % group_name) - -+ if quota.allowed_security_groups(context, 1) < 1: -+ msg = _("Quota exceeded, too many security groups.") -+ raise exception.EC2APIError(msg) -+ - group = {'user_id': context.user_id, - 'project_id': context.project_id, - 'name': group_name, -diff --git a/nova/api/openstack/compute/contrib/quotas.py b/nova/api/openstack/compute/contrib/quotas.py -index 53e8264..6db3d92 100644 ---- a/nova/api/openstack/compute/contrib/quotas.py -+++ b/nova/api/openstack/compute/contrib/quotas.py -@@ -31,7 +31,7 @@ authorize = extensions.extension_authorizer('compute', 'quotas') - - quota_resources = ['metadata_items', 'injected_file_content_bytes', - 'volumes', 'gigabytes', 'ram', 'floating_ips', 'instances', -- 'injected_files', 'cores'] -+ 'injected_files', 'cores', 'security_groups', 'security_group_rules'] - - - class QuotaTemplate(xmlutil.TemplateBuilder): -diff --git a/nova/api/openstack/compute/contrib/security_groups.py b/nova/api/openstack/compute/contrib/security_groups.py -index 0d85c7b..281cc8c 100644 ---- a/nova/api/openstack/compute/contrib/security_groups.py -+++ b/nova/api/openstack/compute/contrib/security_groups.py -@@ -31,6 +31,7 @@ from nova import db - from nova import exception - from nova import flags - from nova import log as logging -+from nova import quota - from nova import utils - - -@@ -289,6 +290,10 @@ class SecurityGroupController(SecurityGroupControllerBase): - group_name = group_name.strip() - group_description = group_description.strip() - -+ if quota.allowed_security_groups(context, 1) < 1: -+ msg = _("Quota exceeded, too many security groups.") -+ raise exc.HTTPBadRequest(explanation=msg) -+ - LOG.audit(_("Create Security Group %s"), group_name, context=context) - self.compute_api.ensure_default_security_group(context) - if db.security_group_exists(context, context.project_id, group_name): -@@ -376,6 +381,13 @@ class SecurityGroupRulesController(SecurityGroupControllerBase): - msg = _('This rule already exists in group %s') % parent_group_id - raise exc.HTTPBadRequest(explanation=msg) - -+ allowed = quota.allowed_security_group_rules(context, -+ parent_group_id, -+ 1) -+ if allowed < 1: -+ msg = _("Quota exceeded, too many security group rules.") -+ raise exc.HTTPBadRequest(explanation=msg) -+ - security_group_rule = db.security_group_rule_create(context, values) - self.sgh.trigger_security_group_rule_create_refresh( - context, [security_group_rule['id']]) -diff --git a/nova/db/api.py b/nova/db/api.py -index b51e1e1..27f80f6 100644 ---- a/nova/db/api.py -+++ b/nova/db/api.py -@@ -1118,6 +1118,11 @@ def security_group_destroy(context, security_group_id): - return IMPL.security_group_destroy(context, security_group_id) - - -+def security_group_count_by_project(context, project_id): -+ """Count number of security groups in a project.""" -+ return IMPL.security_group_count_by_project(context, project_id) -+ -+ - #################### - - -@@ -1149,6 +1154,11 @@ def security_group_rule_get(context, security_group_rule_id): - return IMPL.security_group_rule_get(context, security_group_rule_id) - - -+def security_group_rule_count_by_group(context, security_group_id): -+ """Count rules in a given security group.""" -+ return IMPL.security_group_rule_count_by_group(context, security_group_id) -+ -+ - ################### - - -diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py -index 69e44cd..f2c3062 100644 ---- a/nova/db/sqlalchemy/api.py -+++ b/nova/db/sqlalchemy/api.py -@@ -2813,6 +2813,13 @@ def security_group_destroy(context, security_group_id): - 'updated_at': literal_column('updated_at')}) - - -+@require_context -+def security_group_count_by_project(context, project_id): -+ authorize_project_context(context, project_id) -+ return model_query(context, models.SecurityGroup, read_deleted="no").\ -+ filter_by(project_id=project_id).\ -+ count() -+ - ################### - - -@@ -2871,6 +2878,14 @@ def security_group_rule_destroy(context, security_group_rule_id): - security_group_rule.delete(session=session) - - -+@require_context -+def security_group_rule_count_by_group(context, security_group_id): -+ return model_query(context, models.SecurityGroupIngressRule, -+ read_deleted="no").\ -+ filter_by(parent_group_id=security_group_id).\ -+ count() -+ -+# - ################### - - -@@ -3018,6 +3033,7 @@ def user_update(context, user_id, values): - user_ref.save(session=session) - - -+# - ################### - - -diff --git a/nova/quota.py b/nova/quota.py -index fc49de0..12dd146 100644 ---- a/nova/quota.py -+++ b/nova/quota.py -@@ -54,6 +54,12 @@ quota_opts = [ - cfg.IntOpt('quota_max_injected_file_path_bytes', - default=255, - help='number of bytes allowed per injected file path'), -+ cfg.IntOpt('quota_security_groups', -+ default=10, -+ help='number of security groups per project'), -+ cfg.IntOpt('quota_security_group_rules', -+ default=20, -+ help='number of security rules per security group'), - ] - - FLAGS = flags.FLAGS -@@ -72,6 +78,8 @@ def _get_default_quotas(): - 'injected_files': FLAGS.quota_max_injected_files, - 'injected_file_content_bytes': - FLAGS.quota_max_injected_file_content_bytes, -+ 'security_groups': FLAGS.quota_security_groups, -+ 'security_group_rules': FLAGS.quota_security_group_rules, - } - # -1 in the quota flags means unlimited - for key in defaults.keys(): -@@ -152,6 +160,32 @@ def allowed_floating_ips(context, requested_floating_ips): - return min(requested_floating_ips, allowed_floating_ips) - - -+def allowed_security_groups(context, requested_security_groups): -+ """Check quota and return min(requested, allowed) security groups.""" -+ project_id = context.project_id -+ context = context.elevated() -+ used_sec_groups = db.security_group_count_by_project(context, project_id) -+ quota = get_project_quotas(context, project_id) -+ allowed_sec_groups = _get_request_allotment(requested_security_groups, -+ used_sec_groups, -+ quota['security_groups']) -+ return min(requested_security_groups, allowed_sec_groups) -+ -+ -+def allowed_security_group_rules(context, security_group_id, -+ requested_rules): -+ """Check quota and return min(requested, allowed) sec group rules.""" -+ project_id = context.project_id -+ context = context.elevated() -+ used_rules = db.security_group_rule_count_by_group(context, -+ security_group_id) -+ quota = get_project_quotas(context, project_id) -+ allowed_rules = _get_request_allotment(requested_rules, -+ used_rules, -+ quota['security_group_rules']) -+ return min(requested_rules, allowed_rules) -+ -+ - def _calculate_simple_quota(context, resource, requested): - """Check quota for resource; return min(requested, allowed).""" - quota = get_project_quotas(context, context.project_id) -diff --git a/nova/tests/api/ec2/test_cloud.py b/nova/tests/api/ec2/test_cloud.py -index 9ddc730..427509c 100644 ---- a/nova/tests/api/ec2/test_cloud.py -+++ b/nova/tests/api/ec2/test_cloud.py -@@ -271,6 +271,18 @@ class CloudTestCase(test.TestCase): - delete = self.cloud.delete_security_group - self.assertTrue(delete(self.context, 'testgrp')) - -+ def test_security_group_quota_limit(self): -+ self.flags(quota_security_groups=10) -+ for i in range(1, 10): -+ name = 'test name %i' % i -+ descript = 'test description %i' % i -+ create = self.cloud.create_security_group -+ result = create(self.context, name, descript) -+ -+ # 11'th group should fail -+ self.assertRaises(exception.EC2APIError, -+ create, self.context, 'foo', 'bar') -+ - def test_delete_security_group_by_id(self): - sec = db.security_group_create(self.context, - {'project_id': self.context.project_id, -@@ -436,6 +448,19 @@ class CloudTestCase(test.TestCase): - self.assertRaises(exception.EC2APIError, authz, self.context, - group_name=sec['name'], **kwargs) - -+ def test_security_group_ingress_quota_limit(self): -+ self.flags(quota_security_group_rules=20) -+ kwargs = {'project_id': self.context.project_id, 'name': 'test'} -+ sec_group = db.security_group_create(self.context, kwargs) -+ authz = self.cloud.authorize_security_group_ingress -+ for i in range(100, 120): -+ kwargs = {'to_port': i, 'from_port': i, 'ip_protocol': 'tcp'} -+ authz(self.context, group_id=sec_group['id'], **kwargs) -+ -+ kwargs = {'to_port': 121, 'from_port': 121, 'ip_protocol': 'tcp'} -+ self.assertRaises(exception.EC2APIError, authz, self.context, -+ group_id=sec_group['id'], **kwargs) -+ - def _test_authorize_security_group_no_ports_with_source_group(self, proto): - kwargs = {'project_id': self.context.project_id, 'name': 'test'} - sec = db.security_group_create(self.context, kwargs) -diff --git a/nova/tests/api/openstack/compute/contrib/test_quotas.py b/nova/tests/api/openstack/compute/contrib/test_quotas.py -index 9808717..8f7084a 100644 ---- a/nova/tests/api/openstack/compute/contrib/test_quotas.py -+++ b/nova/tests/api/openstack/compute/contrib/test_quotas.py -@@ -28,7 +28,8 @@ def quota_set(id): - return {'quota_set': {'id': id, 'metadata_items': 128, 'volumes': 10, - 'gigabytes': 1000, 'ram': 51200, 'floating_ips': 10, - 'instances': 10, 'injected_files': 5, 'cores': 20, -- 'injected_file_content_bytes': 10240}} -+ 'injected_file_content_bytes': 10240, -+ 'security_groups': 10, 'security_group_rules': 20}} - - - def quota_set_list(): -@@ -52,7 +53,10 @@ class QuotaSetsTest(test.TestCase): - 'metadata_items': 128, - 'gigabytes': 1000, - 'injected_files': 5, -- 'injected_file_content_bytes': 10240} -+ 'injected_file_content_bytes': 10240, -+ 'security_groups': 10, -+ 'security_group_rules': 20, -+ } - - quota_set = quotas.QuotaSetsController()._format_quota_set('1234', - raw_quota_set) -@@ -68,6 +72,8 @@ class QuotaSetsTest(test.TestCase): - self.assertEqual(qs['metadata_items'], 128) - self.assertEqual(qs['injected_files'], 5) - self.assertEqual(qs['injected_file_content_bytes'], 10240) -+ self.assertEqual(qs['security_groups'], 10) -+ self.assertEqual(qs['security_group_rules'], 20) - - def test_quotas_defaults(self): - uri = '/v2/fake_tenant/os-quota-sets/fake_tenant/defaults' -@@ -85,7 +91,10 @@ class QuotaSetsTest(test.TestCase): - 'floating_ips': 10, - 'metadata_items': 128, - 'injected_files': 5, -- 'injected_file_content_bytes': 10240}} -+ 'injected_file_content_bytes': 10240, -+ 'security_groups': 10, -+ 'security_group_rules': 20, -+ }} - - self.assertEqual(res_dict, expected) - -@@ -106,7 +115,9 @@ class QuotaSetsTest(test.TestCase): - 'ram': 51200, 'volumes': 10, - 'gigabytes': 1000, 'floating_ips': 10, - 'metadata_items': 128, 'injected_files': 5, -- 'injected_file_content_bytes': 10240}} -+ 'injected_file_content_bytes': 10240, -+ 'security_groups': 10, -+ 'security_group_rules': 20}} - - req = fakes.HTTPRequest.blank('/v2/fake4/os-quota-sets/update_me', - use_admin_context=True) -@@ -119,7 +130,9 @@ class QuotaSetsTest(test.TestCase): - 'ram': 51200, 'volumes': 10, - 'gigabytes': 1000, 'floating_ips': 10, - 'metadata_items': 128, 'injected_files': 5, -- 'injected_file_content_bytes': 10240}} -+ 'injected_file_content_bytes': 10240, -+ 'security_groups': 10, -+ 'security_group_rules': 20}} - - req = fakes.HTTPRequest.blank('/v2/fake4/os-quota-sets/update_me') - self.assertRaises(webob.exc.HTTPForbidden, self.controller.update, -@@ -143,6 +156,8 @@ class QuotaXMLSerializerTest(test.TestCase): - floating_ips=60, - instances=70, - injected_files=80, -+ security_groups=10, -+ security_group_rules=20, - cores=90)) - text = self.serializer.serialize(exemplar) - -@@ -166,6 +181,8 @@ class QuotaXMLSerializerTest(test.TestCase): - floating_ips='60', - instances='70', - injected_files='80', -+ security_groups='10', -+ security_group_rules='20', - cores='90')) - intext = ("\n" - '' -@@ -178,6 +195,8 @@ class QuotaXMLSerializerTest(test.TestCase): - '60' - '70' - '80' -+ '10' -+ '20' - '90' - '') - -diff --git a/nova/tests/api/openstack/compute/contrib/test_security_groups.py b/nova/tests/api/openstack/compute/contrib/test_security_groups.py -index 0cf66ec..8cc4cc6 100644 ---- a/nova/tests/api/openstack/compute/contrib/test_security_groups.py -+++ b/nova/tests/api/openstack/compute/contrib/test_security_groups.py -@@ -25,12 +25,15 @@ from nova.api.openstack.compute.contrib import security_groups - from nova.api.openstack import wsgi - import nova.db - from nova import exception -+from nova import flags - from nova import test - from nova.tests.api.openstack import fakes - - - FAKE_UUID = 'a47ae74e-ab08-447f-8eee-ffd43fc46c16' - -+FLAGS = flags.FLAGS -+ - - class AttrDict(dict): - def __getattr__(self, k): -@@ -219,6 +222,18 @@ class TestSecurityGroups(test.TestCase): - self.assertRaises(webob.exc.HTTPBadRequest, self.controller.create, - req, {'security_group': sg}) - -+ def test_create_security_group_quota_limit(self): -+ req = fakes.HTTPRequest.blank('/v2/fake/os-security-groups') -+ for num in range(1, FLAGS.quota_security_groups): -+ name = 'test%s' % num -+ sg = security_group_template(name=name) -+ res_dict = self.controller.create(req, {'security_group': sg}) -+ self.assertEqual(res_dict['security_group']['name'], name) -+ -+ sg = security_group_template() -+ self.assertRaises(webob.exc.HTTPBadRequest, self.controller.create, -+ req, {'security_group': sg}) -+ - def test_get_security_group_list(self): - groups = [] - for i, name in enumerate(['default', 'test']): -@@ -894,6 +909,22 @@ class TestSecurityGroupRules(test.TestCase): - self.assertRaises(webob.exc.HTTPNotFound, self.controller.delete, - req, '22222222222222') - -+ def test_create_rule_quota_limit(self): -+ req = fakes.HTTPRequest.blank('/v2/fake/os-security-group-rules') -+ for num in range(100, 100 + FLAGS.quota_security_group_rules): -+ rule = { -+ 'ip_protocol': 'tcp', 'from_port': num, -+ 'to_port': num, 'parent_group_id': '2', 'group_id': '1' -+ } -+ self.controller.create(req, {'security_group_rule': rule}) -+ -+ rule = { -+ 'ip_protocol': 'tcp', 'from_port': '121', 'to_port': '121', -+ 'parent_group_id': '2', 'group_id': '1' -+ } -+ self.assertRaises(webob.exc.HTTPBadRequest, self.controller.create, -+ req, {'security_group_rule': rule}) -+ - - class TestSecurityGroupRulesXMLDeserializer(unittest.TestCase): - -diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py -index 28c92ca..8cc5577 100644 ---- a/nova/tests/test_quota.py -+++ b/nova/tests/test_quota.py -@@ -235,6 +235,34 @@ class QuotaTestCase(test.TestCase): - floating_ips = quota.allowed_floating_ips(self.context, 101) - self.assertEqual(floating_ips, 101) - -+ def test_unlimited_security_groups(self): -+ self.flags(quota_security_groups=10) -+ security_groups = quota.allowed_security_groups(self.context, 100) -+ self.assertEqual(security_groups, 10) -+ db.quota_create(self.context, self.project_id, 'security_groups', None) -+ security_groups = quota.allowed_security_groups(self.context, 100) -+ self.assertEqual(security_groups, 100) -+ security_groups = quota.allowed_security_groups(self.context, 101) -+ self.assertEqual(security_groups, 101) -+ -+ def test_unlimited_security_group_rules(self): -+ -+ def fake_security_group_rule_count_by_group(context, sec_group_id): -+ return 0 -+ -+ self.stubs.Set(db, 'security_group_rule_count_by_group', -+ fake_security_group_rule_count_by_group) -+ -+ self.flags(quota_security_group_rules=20) -+ rules = quota.allowed_security_group_rules(self.context, 1234, 100) -+ self.assertEqual(rules, 20) -+ db.quota_create(self.context, self.project_id, 'security_group_rules', -+ None) -+ rules = quota.allowed_security_group_rules(self.context, 1234, 100) -+ self.assertEqual(rules, 100) -+ rules = quota.allowed_security_group_rules(self.context, 1234, 101) -+ self.assertEqual(rules, 101) -+ - def test_unlimited_metadata_items(self): - self.flags(quota_metadata_items=10) - items = quota.allowed_metadata_items(self.context, 100) diff --git a/0014-Don-t-leak-RPC-connections-on-timeouts-or-other-exce.patch b/0014-Don-t-leak-RPC-connections-on-timeouts-or-other-exce.patch new file mode 100644 index 0000000..db2b4d3 --- /dev/null +++ b/0014-Don-t-leak-RPC-connections-on-timeouts-or-other-exce.patch @@ -0,0 +1,37 @@ +From 48a07680b46b9973cd7de1b30ae80bd93861e1bb Mon Sep 17 00:00:00 2001 +From: Chris Behrens +Date: Wed, 25 Apr 2012 17:34:53 +0000 +Subject: [PATCH] Don't leak RPC connections on timeouts or other exceptions + +Fixes bug 968843 + +Change-Id: I9e0f1e306cab203bf4c865050b7a45f96127062e +--- + nova/rpc/amqp.py | 7 ++++++- + 1 files changed, 6 insertions(+), 1 deletions(-) + +diff --git a/nova/rpc/amqp.py b/nova/rpc/amqp.py +index 444ade4..4ebd9a4 100644 +--- a/nova/rpc/amqp.py ++++ b/nova/rpc/amqp.py +@@ -39,6 +39,7 @@ from nova import flags + from nova import local + from nova import log as logging + import nova.rpc.common as rpc_common ++from nova import utils + + LOG = logging.getLogger(__name__) + +@@ -296,7 +297,11 @@ class MulticallWaiter(object): + if self._done: + raise StopIteration + while True: +- self._iterator.next() ++ try: ++ self._iterator.next() ++ except Exception: ++ with utils.save_and_reraise_exception(): ++ self.done() + if self._got_ending: + self.done() + raise StopIteration diff --git a/0015-Fixes-bug-987335.patch b/0015-Fixes-bug-987335.patch new file mode 100644 index 0000000..e7737f6 --- /dev/null +++ b/0015-Fixes-bug-987335.patch @@ -0,0 +1,41 @@ +From 108e74b3e770a1d12eda5ed8dea7ca58d5e90cff Mon Sep 17 00:00:00 2001 +From: Alvaro Lopez Garcia +Date: Mon, 23 Apr 2012 16:40:38 +0200 +Subject: [PATCH] Fixes bug 987335. + +Revert bug introduced by commit a837f92e that removed +console_log from get_console_output() + +Change-Id: I22a14b5f50c2df0486420b38137328ac87844c1f +--- + nova/virt/libvirt/connection.py | 10 +++++++--- + 1 files changed, 7 insertions(+), 3 deletions(-) + +diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py +index 888be92..eb0649b 100644 +--- a/nova/virt/libvirt/connection.py ++++ b/nova/virt/libvirt/connection.py +@@ -1006,6 +1006,7 @@ class LibvirtConnection(driver.ComputeDriver): + + self._chown_console_log_for_instance(instance['name']) + data = self._flush_libvirt_console(pty) ++ console_log = self._get_console_log_path(instance_name) + fpath = self._append_to_file(data, console_log) + + return libvirt_utils.load_file(fpath) +@@ -1147,9 +1148,12 @@ class LibvirtConnection(driver.ComputeDriver): + libvirt_utils.mkfs('swap', target) + + @staticmethod +- def _chown_console_log_for_instance(instance_name): +- console_log = os.path.join(FLAGS.instances_path, instance_name, +- 'console.log') ++ def _get_console_log_path(instance_name): ++ return os.path.join(FLAGS.instances_path, instance_name, ++ 'console.log') ++ ++ def _chown_console_log_for_instance(self, instance_name): ++ console_log = self._get_console_log_path(instance_name) + if os.path.exists(console_log): + libvirt_utils.chown(console_log, os.getuid()) + diff --git a/0016-Fix-timeout-in-EC2-CloudController.create_image.patch b/0016-Fix-timeout-in-EC2-CloudController.create_image.patch new file mode 100644 index 0000000..cd705f7 --- /dev/null +++ b/0016-Fix-timeout-in-EC2-CloudController.create_image.patch @@ -0,0 +1,29 @@ +From 1209af45525ed5a58d620a9da92939d39a3d2d9f Mon Sep 17 00:00:00 2001 +From: Eoghan Glynn +Date: Fri, 27 Apr 2012 15:11:57 +0100 +Subject: [PATCH] Fix timeout in EC2 CloudController.create_image() + +Fixes bug 989764 + +The timeout bounding the wait for the instance to stop is intended +to be 1 hour, but the code incorrectly specifies 60 hours instead +(no practical client is going to wait that long for a response). + +Change-Id: I7aa4b539393df15f3b2c950cf7aeca4691ed3d73 +--- + nova/api/ec2/cloud.py | 2 +- + 1 files changed, 1 insertions(+), 1 deletions(-) + +diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py +index 9e2a22e..52def33 100644 +--- a/nova/api/ec2/cloud.py ++++ b/nova/api/ec2/cloud.py +@@ -1614,7 +1614,7 @@ class CloudController(object): + # NOTE(yamahata): timeout and error. 1 hour for now for safety. + # Is it too short/long? + # Or is there any better way? +- timeout = 1 * 60 * 60 * 60 ++ timeout = 1 * 60 * 60 + if time.time() > start_time + timeout: + raise exception.EC2APIError( + _('Couldn\'t stop instance with in %d sec') % timeout) diff --git a/0017-Update-KillFilter-to-handle-deleted-exe-s.patch b/0017-Update-KillFilter-to-handle-deleted-exe-s.patch new file mode 100644 index 0000000..ef4267b --- /dev/null +++ b/0017-Update-KillFilter-to-handle-deleted-exe-s.patch @@ -0,0 +1,61 @@ +From facb936f0bfc6c78fdce93785078e78223b0ddf7 Mon Sep 17 00:00:00 2001 +From: Dan Prince +Date: Wed, 28 Mar 2012 22:00:11 -0400 +Subject: [PATCH] Update KillFilter to handle 'deleted' exe's. + +Updates KillFilter so that it handles the case where the executable +linked to by /proc/PID/exe is updated or deleted. + +Fixes LP Bug #967931. + +Also added a unit test to test that 'deleted' exe's are +filtered correctly. + +(cherry picked from commit b24c11b and commit 3d28e3d) + +Change-Id: I368a01383bf62b64b7579d573b8b84640dec03ae +--- + nova/rootwrap/filters.py | 4 ++++ + nova/tests/test_nova_rootwrap.py | 14 ++++++++++++++ + 2 files changed, 18 insertions(+), 0 deletions(-) + +diff --git a/nova/rootwrap/filters.py b/nova/rootwrap/filters.py +index a8fd513..a51ecae 100755 +--- a/nova/rootwrap/filters.py ++++ b/nova/rootwrap/filters.py +@@ -117,6 +117,10 @@ class KillFilter(CommandFilter): + return False + try: + command = os.readlink("/proc/%d/exe" % int(args[1])) ++ # NOTE(dprince): /proc/PID/exe may have ' (deleted)' on ++ # the end if an executable is updated or deleted ++ if command.endswith(" (deleted)"): ++ command = command[:command.rindex(" ")] + if command not in self.args[1]: + # Affected executable not in accepted list + return False +diff --git a/nova/tests/test_nova_rootwrap.py b/nova/tests/test_nova_rootwrap.py +index ee687ea..ca2626b 100644 +--- a/nova/tests/test_nova_rootwrap.py ++++ b/nova/tests/test_nova_rootwrap.py +@@ -103,6 +103,20 @@ class RootwrapTestCase(test.TestCase): + usercmd = ['kill', 'notapid'] + self.assertFalse(f.match(usercmd)) + ++ def test_KillFilter_deleted_exe(self): ++ """Makes sure deleted exe's are killed correctly""" ++ # See bug #967931. ++ def fake_readlink(blah): ++ return '/bin/commandddddd (deleted)' ++ ++ f = filters.KillFilter("/bin/kill", "root", ++ [""], ++ ["/bin/commandddddd"]) ++ usercmd = ['kill', 1234] ++ # Providing no signal should work ++ self.stubs.Set(os, 'readlink', fake_readlink) ++ self.assertTrue(f.match(usercmd)) ++ + def test_ReadFileFilter(self): + goodfn = '/good/file.name' + f = filters.ReadFileFilter(goodfn) diff --git a/0018-Get-unit-tests-functional-in-OS-X.patch b/0018-Get-unit-tests-functional-in-OS-X.patch new file mode 100644 index 0000000..4e381ff --- /dev/null +++ b/0018-Get-unit-tests-functional-in-OS-X.patch @@ -0,0 +1,51 @@ +From 76b525ab22ac63282153e5a7eb9cf5947da10413 Mon Sep 17 00:00:00 2001 +From: Matt Stephenson +Date: Tue, 3 Apr 2012 14:38:09 -0700 +Subject: [PATCH] Get unit tests functional in OS X + +* Add detection for directio to ensure the python runtime is built with O_DIRECT +* Extend stubbing in test_libvirt to also stub out _supports_direct_io + +Change-Id: Id793d4039311396f0b3c3a52d2a1d951ec3c5e48 +(cherry picked from commit cf7c0a7c10723495953be9bf99aedbe3838e0787) +--- + nova/tests/test_libvirt.py | 7 +++++++ + nova/virt/libvirt/connection.py | 6 ++++++ + 2 files changed, 13 insertions(+), 0 deletions(-) + +diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py +index cdc9121..5163b32 100644 +--- a/nova/tests/test_libvirt.py ++++ b/nova/tests/test_libvirt.py +@@ -899,6 +899,13 @@ class LibvirtConnTestCase(test.TestCase): + + self.stubs.Set(os, 'open', os_open_stub) + ++ def connection_supports_direct_io_stub(*args, **kwargs): ++ return directio_supported ++ ++ self.stubs.Set(connection.LibvirtConnection, ++ '_supports_direct_io', ++ connection_supports_direct_io_stub) ++ + user_context = context.RequestContext(self.user_id, self.project_id) + instance_ref = db.instance_create(user_context, self.test_instance) + network_info = _fake_network_info(self.stubs, 1) +diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py +index 5facc10..00345d2 100644 +--- a/nova/virt/libvirt/connection.py ++++ b/nova/virt/libvirt/connection.py +@@ -1037,7 +1037,13 @@ class LibvirtConnection(driver.ComputeDriver): + + @staticmethod + def _supports_direct_io(dirpath): ++ ++ if not hasattr(os, 'O_DIRECT'): ++ LOG.debug("This python runtime does not support direct I/O") ++ return False ++ + testfile = os.path.join(dirpath, ".directio.test") ++ + hasDirectIO = True + try: + f = os.open(testfile, os.O_CREAT | os.O_WRONLY | os.O_DIRECT) diff --git a/0019-Introduced-flag-base_dir_name.-Fixes-bug-973194.patch b/0019-Introduced-flag-base_dir_name.-Fixes-bug-973194.patch new file mode 100644 index 0000000..094f4d9 --- /dev/null +++ b/0019-Introduced-flag-base_dir_name.-Fixes-bug-973194.patch @@ -0,0 +1,179 @@ +From 7028d66ae97c68f888a2bbf2d3b431702f72b4c5 Mon Sep 17 00:00:00 2001 +From: Mandar Vaze +Date: Thu, 5 Apr 2012 01:33:34 -0700 +Subject: [PATCH] Introduced flag base_dir_name. Fixes bug 973194 + +rebased from master. + +If user faces locking related problem when two nova-compute hosts +sharing same disk area via nfs, try to download same image into +cache concurrently - Then base_dir_name can be set to "_base_$my_ip" in +nova.conf + +Default value for base_dir_name is "_base" thus retaining existing +behavior. + +Change-Id: Icff10ed75ba83f7256731614dc9e01e578b347a4 +--- + Authors | 1 + + nova/compute/manager.py | 6 ++++++ + nova/tests/test_imagecache.py | 8 +++++--- + nova/tests/test_libvirt.py | 7 ++++--- + nova/virt/libvirt/connection.py | 3 ++- + nova/virt/libvirt/imagecache.py | 6 ++++-- + 6 files changed, 22 insertions(+), 9 deletions(-) + +diff --git a/Authors b/Authors +index a229313..b9ad28b 100644 +--- a/Authors ++++ b/Authors +@@ -122,6 +122,7 @@ Likitha Shetty + Loganathan Parthipan + Lorin Hochstein + Lvov Maxim ++Mandar Vaze + Mandell Degerness + Mark McClain + Mark McLoughlin +diff --git a/nova/compute/manager.py b/nova/compute/manager.py +index 48e135b..053e80e 100644 +--- a/nova/compute/manager.py ++++ b/nova/compute/manager.py +@@ -28,6 +28,7 @@ terminating it. + **Related Flags** + + :instances_path: Where instances are kept on disk ++:base_dir_name: Where cached images are stored under instances_path + :compute_driver: Name of class that is used to handle virtualization, loaded + by :func:`nova.utils.import_object` + +@@ -72,6 +73,11 @@ compute_opts = [ + cfg.StrOpt('instances_path', + default='$state_path/instances', + help='where instances are stored on disk'), ++ cfg.StrOpt('base_dir_name', ++ default='_base', ++ help="where cached images are stored under $instances_path" ++ "This is NOT full path - just a folder name" ++ "For per-compute-host cached images, Set to _base_$my_ip"), + cfg.StrOpt('compute_driver', + default='nova.virt.connection.get_connection', + help='Driver to use for controlling virtualization'), +diff --git a/nova/tests/test_imagecache.py b/nova/tests/test_imagecache.py +index 9cf4003..f1d5aa5 100644 +--- a/nova/tests/test_imagecache.py ++++ b/nova/tests/test_imagecache.py +@@ -36,6 +36,7 @@ from nova.virt.libvirt import utils as virtutils + + + flags.DECLARE('instances_path', 'nova.compute.manager') ++flags.DECLARE('base_dir_name', 'nova.compute.manager') + FLAGS = flags.FLAGS + + LOG = log.getLogger(__name__) +@@ -155,7 +156,7 @@ class ImageCacheManagerTestCase(test.TestCase): + self.stubs.Set(virtutils, 'get_disk_backing_file', + lambda x: 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm') + +- found = os.path.join(FLAGS.instances_path, '_base', ++ found = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name, + 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm') + + image_cache_manager = imagecache.ImageCacheManager() +@@ -177,7 +178,7 @@ class ImageCacheManagerTestCase(test.TestCase): + lambda x: ('e97222e91fc4241f49a7f520d1dcf446751129b3_' + '10737418240')) + +- found = os.path.join(FLAGS.instances_path, '_base', ++ found = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name, + 'e97222e91fc4241f49a7f520d1dcf446751129b3_' + '10737418240') + +@@ -198,7 +199,7 @@ class ImageCacheManagerTestCase(test.TestCase): + self.stubs.Set(virtutils, 'get_disk_backing_file', + lambda x: 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm') + +- found = os.path.join(FLAGS.instances_path, '_base', ++ found = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name, + 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm') + + image_cache_manager = imagecache.ImageCacheManager() +@@ -521,6 +522,7 @@ class ImageCacheManagerTestCase(test.TestCase): + hashed_42 = '92cfceb39d57d914ed8b14d0e37643de0797ae56' + + self.flags(instances_path='/instance_path') ++ self.flags(base_dir_name='_base') + self.flags(remove_unused_base_images=True) + + base_file_list = ['00000001', +diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py +index cdc9121..f12cad4 100644 +--- a/nova/tests/test_libvirt.py ++++ b/nova/tests/test_libvirt.py +@@ -322,7 +322,7 @@ class CacheConcurrencyTestCase(test.TestCase): + self.flags(instances_path='nova.compute.manager') + + def fake_exists(fname): +- basedir = os.path.join(FLAGS.instances_path, '_base') ++ basedir = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name) + if fname == basedir: + return True + return False +@@ -1317,9 +1317,10 @@ class LibvirtConnTestCase(test.TestCase): + if os.path.isdir(path): + shutil.rmtree(path) + +- path = os.path.join(FLAGS.instances_path, '_base') ++ path = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name) + if os.path.isdir(path): +- shutil.rmtree(os.path.join(FLAGS.instances_path, '_base')) ++ shutil.rmtree(os.path.join(FLAGS.instances_path, ++ FLAGS.base_dir_name)) + + def test_get_host_ip_addr(self): + conn = connection.LibvirtConnection(False) +diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py +index 5facc10..f1e5680 100644 +--- a/nova/virt/libvirt/connection.py ++++ b/nova/virt/libvirt/connection.py +@@ -1090,7 +1090,8 @@ class LibvirtConnection(driver.ComputeDriver): + + generating = 'image_id' not in kwargs + if not os.path.exists(target): +- base_dir = os.path.join(FLAGS.instances_path, '_base') ++ base_dir = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name) ++ + if not os.path.exists(base_dir): + libvirt_utils.ensure_tree(base_dir) + base = os.path.join(base_dir, fname) +diff --git a/nova/virt/libvirt/imagecache.py b/nova/virt/libvirt/imagecache.py +index f92376a..adf8214 100644 +--- a/nova/virt/libvirt/imagecache.py ++++ b/nova/virt/libvirt/imagecache.py +@@ -58,6 +58,7 @@ imagecache_opts = [ + ] + + flags.DECLARE('instances_path', 'nova.compute.manager') ++flags.DECLARE('base_dir_name', 'nova.compute.manager') + FLAGS = flags.FLAGS + FLAGS.register_opts(imagecache_opts) + +@@ -178,7 +179,8 @@ class ImageCacheManager(object): + 'backing': backing_file}) + + backing_path = os.path.join(FLAGS.instances_path, +- '_base', backing_file) ++ FLAGS.base_dir_name, ++ backing_file) + if not backing_path in inuse_images: + inuse_images.append(backing_path) + +@@ -372,7 +374,7 @@ class ImageCacheManager(object): + # created, but may remain from previous versions. + self._reset_state() + +- base_dir = os.path.join(FLAGS.instances_path, '_base') ++ base_dir = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name) + if not os.path.exists(base_dir): + LOG.debug(_('Skipping verification, no base directory at %s'), + base_dir) diff --git a/0020-Fix-bug-983206-_try_convert-parsing-string.patch b/0020-Fix-bug-983206-_try_convert-parsing-string.patch new file mode 100644 index 0000000..7b4a028 --- /dev/null +++ b/0020-Fix-bug-983206-_try_convert-parsing-string.patch @@ -0,0 +1,103 @@ +From 21e918a8f6e0fd144287ff7fc2ab3d262ac9edd7 Mon Sep 17 00:00:00 2001 +From: Joe Gordon +Date: Fri, 13 Apr 2012 15:12:04 -0400 +Subject: [PATCH] Fix bug 983206 : _try_convert parsing string + +* _try_convert in ec2utils.py didn't handle strings starting with "0x" +* Added tests to cover bug +* Add better float support +* remove unused complex number support + +Change-Id: I382d36f4a8671bcceccfa1ebdbae89a9d2aca207 +(cherry picked from commit c95162e52899618fc269fb536f6a2d3b26b7794d) +--- + nova/api/ec2/ec2utils.py | 35 +++++++++++------------------------ + nova/tests/test_api.py | 12 ++++++++++++ + 2 files changed, 23 insertions(+), 24 deletions(-) + +diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py +index 1a6bb96..0f4aeb0 100644 +--- a/nova/api/ec2/ec2utils.py ++++ b/nova/api/ec2/ec2utils.py +@@ -166,6 +166,10 @@ def _try_convert(value): + * try conversion to int, float, complex, fallback value + + """ ++ def _negative_zero(value): ++ epsilon = 1e-7 ++ return 0 if abs(value) < epsilon else value ++ + if len(value) == 0: + return '' + if value == 'None': +@@ -175,31 +179,14 @@ def _try_convert(value): + return True + if lowered_value == 'false': + return False +- valueneg = value[1:] if value[0] == '-' else value +- if valueneg == '0': +- return 0 +- if valueneg == '': +- return value +- if valueneg[0] == '0': +- if valueneg[1] in 'xX': +- return int(value, 16) +- elif valueneg[1] in 'bB': +- return int(value, 2) +- else: +- try: +- return int(value, 8) +- except ValueError: +- pass +- try: +- return int(value) +- except ValueError: +- pass +- try: +- return float(value) +- except ValueError: +- pass ++ for prefix, base in [('0x', 16), ('0b', 2), ('0', 8), ('', 10)]: ++ try: ++ if lowered_value.startswith((prefix, "-" + prefix)): ++ return int(lowered_value, base) ++ except ValueError: ++ pass + try: +- return complex(value) ++ return _negative_zero(float(value)) + except ValueError: + return value + +diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py +index baaee98..a52319f 100644 +--- a/nova/tests/test_api.py ++++ b/nova/tests/test_api.py +@@ -96,8 +96,10 @@ class XmlConversionTestCase(test.TestCase): + conv = ec2utils._try_convert + self.assertEqual(conv('None'), None) + self.assertEqual(conv('True'), True) ++ self.assertEqual(conv('TRUE'), True) + self.assertEqual(conv('true'), True) + self.assertEqual(conv('False'), False) ++ self.assertEqual(conv('FALSE'), False) + self.assertEqual(conv('false'), False) + self.assertEqual(conv('0'), 0) + self.assertEqual(conv('42'), 42) +@@ -107,6 +109,16 @@ class XmlConversionTestCase(test.TestCase): + self.assertEqual(conv('-0x57'), -0x57) + self.assertEqual(conv('-'), '-') + self.assertEqual(conv('-0'), 0) ++ self.assertEqual(conv('0.0'), 0.0) ++ self.assertEqual(conv('1e-8'), 0.0) ++ self.assertEqual(conv('-1e-8'), 0.0) ++ self.assertEqual(conv('0xDD8G'), '0xDD8G') ++ self.assertEqual(conv('0XDD8G'), '0XDD8G') ++ self.assertEqual(conv('-stringy'), '-stringy') ++ self.assertEqual(conv('stringy'), 'stringy') ++ self.assertEqual(conv('add'), 'add') ++ self.assertEqual(conv('remove'), 'remove') ++ self.assertEqual(conv(''), '') + + + class Ec2utilsTestCase(test.TestCase): diff --git a/0021-QuantumManager-will-start-dnsmasq-during-startup.-Fi.patch b/0021-QuantumManager-will-start-dnsmasq-during-startup.-Fi.patch new file mode 100644 index 0000000..3f37ff0 --- /dev/null +++ b/0021-QuantumManager-will-start-dnsmasq-during-startup.-Fi.patch @@ -0,0 +1,50 @@ +From 26dc6b75c73f10c2da7628ce59e225d1006d9d1c Mon Sep 17 00:00:00 2001 +From: Mandar Vaze +Date: Wed, 11 Apr 2012 01:43:22 -0700 +Subject: [PATCH] QuantumManager will start dnsmasq during startup. Fixes bug + 977759 + +Added _setup_network_on_host method, which calls update_dhcp +if quantum_use_dhcp is set. + +Change-Id: I193212037873001a03da7b7a484f61a5c13b5de8 +--- + Authors | 1 + + nova/network/quantum/manager.py | 17 +++++++++++++++++ + 2 files changed, 18 insertions(+), 0 deletions(-) + +diff --git a/nova/network/quantum/manager.py b/nova/network/quantum/manager.py +index eb0f389..498b5f0 100644 +--- a/nova/network/quantum/manager.py ++++ b/nova/network/quantum/manager.py +@@ -88,6 +88,7 @@ class QuantumManager(manager.FloatingIP, manager.FlatManager): + def init_host(self): + # Initialize general L3 networking + self.l3driver.initialize() ++ super(QuantumManager, self).init_host() + # Initialize floating ip support (only works for nova ipam currently) + if FLAGS.quantum_ipam_lib == 'nova.network.quantum.nova_ipam_lib': + LOG.debug("Initializing FloatingIP support") +@@ -107,6 +108,22 @@ class QuantumManager(manager.FloatingIP, manager.FlatManager): + for c in cidrs: + self.l3driver.initialize_network(c) + ++ # Similar to FlatDHCPMananger, except we check for quantum_use_dhcp flag ++ # before we try to update_dhcp ++ def _setup_network_on_host(self, context, network): ++ """Sets up network on this host.""" ++ network['dhcp_server'] = self._get_dhcp_ip(context, network) ++ self.l3driver.initialize_gateway(network) ++ ++ if FLAGS.quantum_use_dhcp and not FLAGS.fake_network: ++ dev = self.driver.get_dev(network) ++ self.driver.update_dhcp(context, dev, network) ++ if FLAGS.use_ipv6: ++ self.driver.update_ra(context, dev, network) ++ gateway = utils.get_my_linklocal(dev) ++ self.db.network_update(context, network['id'], ++ {'gateway_v6': gateway}) ++ + def _update_network_host(self, context, net_uuid): + """Set the host column in the networks table: note that this won't + work with multi-host but QuantumManager doesn't support that diff --git a/0022-Fixes-bug-952176.patch b/0022-Fixes-bug-952176.patch new file mode 100644 index 0000000..2447930 --- /dev/null +++ b/0022-Fixes-bug-952176.patch @@ -0,0 +1,28 @@ +From 9e9a554cba9e52430c2b2857bed744aba2ff8f9e Mon Sep 17 00:00:00 2001 +From: MotoKen +Date: Mon, 9 Apr 2012 10:33:55 +0800 +Subject: [PATCH] Fixes bug 952176 + +Checks if value is string or not before decode. + +Change-Id: I3f839770fdd7b00223ce02b95b2a265d903fa00e +--- + bin/nova-manage | 4 +++- + 1 files changed, 3 insertions(+), 1 deletions(-) + +diff --git a/bin/nova-manage b/bin/nova-manage +index c0009bc..f5491bc 100755 +--- a/bin/nova-manage ++++ b/bin/nova-manage +@@ -1721,8 +1721,10 @@ def main(): + for k, v in fn_kwargs.items(): + if v is None: + del fn_kwargs[k] +- else: ++ elif isinstance(v, basestring): + fn_kwargs[k] = v.decode('utf-8') ++ else: ++ fn_kwargs[k] = v + + fn_args = [arg.decode('utf-8') for arg in fn_args] + diff --git a/0023-Fix-nova.tests.test_nova_rootwrap-on-Fedora-17.patch b/0023-Fix-nova.tests.test_nova_rootwrap-on-Fedora-17.patch new file mode 100644 index 0000000..c95fcf3 --- /dev/null +++ b/0023-Fix-nova.tests.test_nova_rootwrap-on-Fedora-17.patch @@ -0,0 +1,38 @@ +From e5e890f3117c792544d6a87d887543d502d1cb55 Mon Sep 17 00:00:00 2001 +From: Russell Bryant +Date: Tue, 1 May 2012 18:29:04 -0400 +Subject: [PATCH] Fix nova.tests.test_nova_rootwrap on Fedora 17. + +Fix bug 992916 + +This patch resolves a unit test failure on Fedora 17. The root cause is +that 'sleep' is '/usr/bin/sleep' instead of '/bin/sleep'. Update the +test to allow that. + +Change-Id: I5c8e04baec7159a8c10c9beb96cff58fd383e71c +--- + nova/tests/test_nova_rootwrap.py | 4 ++-- + 1 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/nova/tests/test_nova_rootwrap.py b/nova/tests/test_nova_rootwrap.py +index ca2626b..4cd6818 100644 +--- a/nova/tests/test_nova_rootwrap.py ++++ b/nova/tests/test_nova_rootwrap.py +@@ -69,7 +69,7 @@ class RootwrapTestCase(test.TestCase): + p = subprocess.Popen(["/bin/sleep", "5"]) + f = filters.KillFilter("/bin/kill", "root", + ["-ALRM"], +- ["/bin/sleep"]) ++ ["/bin/sleep", "/usr/bin/sleep"]) + usercmd = ['kill', '-9', p.pid] + # Incorrect signal should fail + self.assertFalse(f.match(usercmd)) +@@ -79,7 +79,7 @@ class RootwrapTestCase(test.TestCase): + + f = filters.KillFilter("/bin/kill", "root", + ["-9", ""], +- ["/bin/sleep"]) ++ ["/bin/sleep", "/usr/bin/sleep"]) + usercmd = ['kill', '-9', os.getpid()] + # Our own PID does not match /bin/sleep, so it should fail + self.assertFalse(f.match(usercmd)) diff --git a/0024-ensure-atomic-manipulation-of-libvirt-disk-images.patch b/0024-ensure-atomic-manipulation-of-libvirt-disk-images.patch new file mode 100644 index 0000000..f4a760f --- /dev/null +++ b/0024-ensure-atomic-manipulation-of-libvirt-disk-images.patch @@ -0,0 +1,218 @@ +From 6a3eabcd01981c6ccead47e2b610bd82b5d6be80 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?P=C3=A1draig=20Brady?= +Date: Fri, 16 Mar 2012 03:43:49 +0000 +Subject: [PATCH] ensure atomic manipulation of libvirt disk images + +This pattern could probably be used elsewhere, +but only libvirt disk images are considered for now. +This change ensures there are no stale files left +anywhere in the path from glance, through the libvirt image cache. +These could cause subsequent operational errors either +directly or indirectly through disk wastage. + +* nova/utils.py: Add a new remove_path_on_error() context manager +that is used to remove the passed PATH on a raised exception. +* nova/virt/images.py: Ensure temporary downloaded and +converted images are protected. +* nova/virt/libvirt/connection.py: Ensure all the images in +the image cache and instance dirs are protected. + +Change-Id: I81a5407665a6998128c0dee41387ef00ebddeb4d +--- + nova/utils.py | 21 +++++++++-- + nova/virt/images.py | 69 +++++++++++++++++---------------------- + nova/virt/libvirt/connection.py | 16 ++++++--- + 3 files changed, 57 insertions(+), 49 deletions(-) + +diff --git a/nova/utils.py b/nova/utils.py +index 819929a..7188d98 100644 +--- a/nova/utils.py ++++ b/nova/utils.py +@@ -21,6 +21,7 @@ + + import contextlib + import datetime ++import errno + import functools + import hashlib + import inspect +@@ -1013,8 +1014,8 @@ def cleanup_file_locks(): + continue + try: + stat_info = os.stat(os.path.join(FLAGS.lock_path, filename)) +- except OSError as (errno, strerror): +- if errno == 2: # doesn't exist ++ except OSError as e: ++ if e.errno == errno.ENOENT: + continue + else: + raise +@@ -1033,8 +1034,8 @@ def delete_if_exists(pathname): + + try: + os.unlink(pathname) +- except OSError as (errno, strerror): +- if errno == 2: # doesn't exist ++ except OSError as e: ++ if e.errno == errno.ENOENT: + return + else: + raise +@@ -1344,6 +1345,18 @@ def logging_error(message): + LOG.exception(message) + + ++@contextlib.contextmanager ++def remove_path_on_error(path): ++ """Protect code that wants to operate on PATH atomically. ++ Any exception will cause PATH to be removed. ++ """ ++ try: ++ yield ++ except Exception: ++ with save_and_reraise_exception(): ++ delete_if_exists(path) ++ ++ + def make_dev_path(dev, partition=None, base='/dev'): + """Return a path to a particular device. + +diff --git a/nova/virt/images.py b/nova/virt/images.py +index 1e0ae0a..626f3ff 100644 +--- a/nova/virt/images.py ++++ b/nova/virt/images.py +@@ -51,18 +51,10 @@ def fetch(context, image_href, path, _user_id, _project_id): + # checked before we got here. + (image_service, image_id) = nova.image.get_image_service(context, + image_href) +- try: ++ with utils.remove_path_on_error(path): + with open(path, "wb") as image_file: + metadata = image_service.get(context, image_id, image_file) +- except Exception: +- with utils.save_and_reraise_exception(): +- try: +- os.unlink(path) +- except OSError, e: +- if e.errno != errno.ENOENT: +- LOG.warn("unable to remove stale image '%s': %s" % +- (path, e.strerror)) +- return metadata ++ return metadata + + + def fetch_to_raw(context, image_href, path, user_id, project_id): +@@ -85,37 +77,36 @@ def fetch_to_raw(context, image_href, path, user_id, project_id): + + return(data) + +- data = _qemu_img_info(path_tmp) +- +- fmt = data.get("file format") +- if fmt is None: +- os.unlink(path_tmp) +- raise exception.ImageUnacceptable( +- reason=_("'qemu-img info' parsing failed."), image_id=image_href) +- +- if "backing file" in data: +- backing_file = data['backing file'] +- os.unlink(path_tmp) +- raise exception.ImageUnacceptable(image_id=image_href, +- reason=_("fmt=%(fmt)s backed by: %(backing_file)s") % locals()) +- +- if fmt != "raw" and FLAGS.force_raw_images: +- staged = "%s.converted" % path +- LOG.debug("%s was %s, converting to raw" % (image_href, fmt)) +- out, err = utils.execute('qemu-img', 'convert', '-O', 'raw', +- path_tmp, staged) +- os.unlink(path_tmp) +- +- data = _qemu_img_info(staged) +- if data.get('file format', None) != "raw": +- os.unlink(staged) ++ with utils.remove_path_on_error(path_tmp): ++ data = _qemu_img_info(path_tmp) ++ ++ fmt = data.get("file format") ++ if fmt is None: ++ raise exception.ImageUnacceptable( ++ reason=_("'qemu-img info' parsing failed."), ++ image_id=image_href) ++ ++ if "backing file" in data: ++ backing_file = data['backing file'] + raise exception.ImageUnacceptable(image_id=image_href, +- reason=_("Converted to raw, but format is now %s") % +- data.get('file format', None)) ++ reason=_("fmt=%(fmt)s backed by: %(backing_file)s") % locals()) ++ ++ if fmt != "raw" and FLAGS.force_raw_images: ++ staged = "%s.converted" % path ++ LOG.debug("%s was %s, converting to raw" % (image_href, fmt)) ++ with utils.remove_path_on_error(staged): ++ out, err = utils.execute('qemu-img', 'convert', '-O', 'raw', ++ path_tmp, staged) ++ ++ data = _qemu_img_info(staged) ++ if data.get('file format', None) != "raw": ++ raise exception.ImageUnacceptable(image_id=image_href, ++ reason=_("Converted to raw, but format is now %s") % ++ data.get('file format', None)) + +- os.rename(staged, path) ++ os.rename(staged, path) + +- else: +- os.rename(path_tmp, path) ++ else: ++ os.rename(path_tmp, path) + + return metadata +diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py +index 31e6511..dc16d05 100644 +--- a/nova/virt/libvirt/connection.py ++++ b/nova/virt/libvirt/connection.py +@@ -1105,7 +1105,8 @@ class LibvirtConnection(driver.ComputeDriver): + @utils.synchronized(fname) + def call_if_not_exists(base, fn, *args, **kwargs): + if not os.path.exists(base): +- fn(target=base, *args, **kwargs) ++ with utils.remove_path_on_error(base): ++ fn(target=base, *args, **kwargs) + + if cow or not generating: + call_if_not_exists(base, fn, *args, **kwargs) +@@ -1121,8 +1122,9 @@ class LibvirtConnection(driver.ComputeDriver): + size_gb = size / (1024 * 1024 * 1024) + cow_base += "_%d" % size_gb + if not os.path.exists(cow_base): +- libvirt_utils.copy_image(base, cow_base) +- disk.extend(cow_base, size) ++ with utils.remove_path_on_error(cow_base): ++ libvirt_utils.copy_image(base, cow_base) ++ disk.extend(cow_base, size) + libvirt_utils.create_cow_image(cow_base, target) + elif not generating: + libvirt_utils.copy_image(base, target) +@@ -1132,7 +1134,8 @@ class LibvirtConnection(driver.ComputeDriver): + if size: + disk.extend(target, size) + +- copy_and_extend(cow, generating, base, target, size) ++ with utils.remove_path_on_error(target): ++ copy_and_extend(cow, generating, base, target, size) + + @staticmethod + def _create_local(target, local_size, unit='G', +@@ -1305,8 +1308,9 @@ class LibvirtConnection(driver.ComputeDriver): + project_id=instance['project_id'],) + elif config_drive: + label = 'config' +- self._create_local(basepath('disk.config'), 64, unit='M', +- fs_format='msdos', label=label) # 64MB ++ with utils.remove_path_on_error(basepath('disk.config')): ++ self._create_local(basepath('disk.config'), 64, unit='M', ++ fs_format='msdos', label=label) # 64MB + + if FLAGS.libvirt_inject_key and instance['key_data']: + key = str(instance['key_data']) diff --git a/0025-Ensure-we-don-t-access-the-net-when-building-docs.patch b/0025-Ensure-we-don-t-access-the-net-when-building-docs.patch new file mode 100644 index 0000000..0f49ded --- /dev/null +++ b/0025-Ensure-we-don-t-access-the-net-when-building-docs.patch @@ -0,0 +1,25 @@ +From 73185a4a4abe3dc87efa7ec1b4e60f98c049b75b Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?P=C3=A1draig=20Brady?= +Date: Fri, 6 Jan 2012 12:16:34 +0000 +Subject: [PATCH] Ensure we don't access the net when building docs + +(Note, this has not been sent upstream) + +Change-Id: I9d02fb4053a8106672aded1614a2850e21603eb2 +--- + doc/source/conf.py | 2 +- + 1 files changed, 1 insertions(+), 1 deletions(-) + +diff --git a/doc/source/conf.py b/doc/source/conf.py +index 8ced294..7df59cd 100644 +--- a/doc/source/conf.py ++++ b/doc/source/conf.py +@@ -25,7 +25,7 @@ sys.path.insert(0, os.path.abspath('./')) + # Add any Sphinx extension module names here, as strings. They can be extensions + # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. + +-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'ext.nova_todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig','sphinx.ext.graphviz'] ++extensions = ['sphinx.ext.autodoc', 'ext.nova_todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig','sphinx.ext.graphviz'] + + # autodoc generation is a bit aggressive and a nuisance when doing heavy text edit cycles. + # execute "export SPHINX_DEBUG=1" in your terminal to disable diff --git a/0026-fix-useexisting-deprecation-warnings.patch b/0026-fix-useexisting-deprecation-warnings.patch new file mode 100644 index 0000000..dbc3a07 --- /dev/null +++ b/0026-fix-useexisting-deprecation-warnings.patch @@ -0,0 +1,49 @@ +From bf7f18bf91718babb30e8ded89410667bc940320 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?P=C3=A1draig=20Brady?= +Date: Thu, 8 Mar 2012 16:32:30 +0000 +Subject: [PATCH] fix useexisting deprecation warnings + +Fixes deprecation warnings when using sqlalchemy >= 0.7.0 +Fixes bug 941951 + +Change-Id: Iaa57153f99c60c67a14c1dca849188937bdc5dee +--- + .../075_convert_bw_usage_to_store_network_id.py | 4 ++-- + .../versions/081_drop_instance_id_bw_cache.py | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/nova/db/sqlalchemy/migrate_repo/versions/075_convert_bw_usage_to_store_network_id.py b/nova/db/sqlalchemy/migrate_repo/versions/075_convert_bw_usage_to_store_network_id.py +index b275524..4ff3d99 100644 +--- a/nova/db/sqlalchemy/migrate_repo/versions/075_convert_bw_usage_to_store_network_id.py ++++ b/nova/db/sqlalchemy/migrate_repo/versions/075_convert_bw_usage_to_store_network_id.py +@@ -46,7 +46,7 @@ def upgrade(migrate_engine): + Column('last_refreshed', DateTime(timezone=False)), + Column('bw_in', BigInteger()), + Column('bw_out', BigInteger()), +- useexisting=True) ++ extend_existing=True) + mac_column = Column('mac', String(255)) + bw_usage_cache.create_column(mac_column) + +@@ -81,7 +81,7 @@ def downgrade(migrate_engine): + Column('last_refreshed', DateTime(timezone=False)), + Column('bw_in', BigInteger()), + Column('bw_out', BigInteger()), +- useexisting=True) ++ extend_existing=True) + + network_label_column = Column('network_label', String(255)) + bw_usage_cache.create_column(network_label_column) +diff --git a/nova/db/sqlalchemy/migrate_repo/versions/081_drop_instance_id_bw_cache.py b/nova/db/sqlalchemy/migrate_repo/versions/081_drop_instance_id_bw_cache.py +index c6687ac..a607ed3 100644 +--- a/nova/db/sqlalchemy/migrate_repo/versions/081_drop_instance_id_bw_cache.py ++++ b/nova/db/sqlalchemy/migrate_repo/versions/081_drop_instance_id_bw_cache.py +@@ -37,7 +37,7 @@ def upgrade(migrate_engine): + Column('last_refreshed', DateTime(timezone=False)), + Column('bw_in', BigInteger()), + Column('bw_out', BigInteger()), +- useexisting=True) ++ extend_existing=True) + + bw_usage_cache.drop_column('instance_id') + diff --git a/0027-support-a-configurable-libvirt-injection-partition.patch b/0027-support-a-configurable-libvirt-injection-partition.patch new file mode 100644 index 0000000..343b50e --- /dev/null +++ b/0027-support-a-configurable-libvirt-injection-partition.patch @@ -0,0 +1,80 @@ +From 862cb7a4bad82f7347f495ad3a91df31cad79214 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?P=C3=A1draig=20Brady?= +Date: Wed, 18 Apr 2012 23:27:31 +0100 +Subject: [PATCH] support a configurable libvirt injection partition + +This is useful if all guest images have the same structure, +and the root partition is not the first partition. + +This is also handy to enable inspection in libguestfs, +which can handle disparate and complicated image layouts. + +In future we may change to a StrOpt to support +searching by partition label. + +Change-Id: Ie94d61bec8fe4b41d6d2d6d3efa9a4364cf027fe + +Conflicts: + + nova/virt/libvirt/connection.py +--- + nova/virt/disk/mount.py | 6 ++++-- + nova/virt/libvirt/connection.py | 12 ++++++++---- + 2 files changed, 12 insertions(+), 6 deletions(-) + +diff --git a/nova/virt/disk/mount.py b/nova/virt/disk/mount.py +index 4fb5dda..11959b2 100644 +--- a/nova/virt/disk/mount.py ++++ b/nova/virt/disk/mount.py +@@ -58,7 +58,9 @@ class Mount(object): + """Map partitions of the device to the file system namespace.""" + assert(os.path.exists(self.device)) + +- if self.partition: ++ if self.partition == -1: ++ self.error = _('partition search unsupported with %s') % self.mode ++ elif self.partition: + map_path = '/dev/mapper/%sp%s' % (os.path.basename(self.device), + self.partition) + assert(not os.path.exists(map_path)) +@@ -73,7 +75,7 @@ class Mount(object): + # so given we only use it when we expect a partitioned image, fail + if not os.path.exists(map_path): + if not err: +- err = _('no partitions found') ++ err = _('partition %s not found') % self.partition + self.error = _('Failed to map partitions: %s') % err + else: + self.mapped_device = map_path +diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py +index dc16d05..81fd587 100644 +--- a/nova/virt/libvirt/connection.py ++++ b/nova/virt/libvirt/connection.py +@@ -108,6 +108,11 @@ libvirt_opts = [ + cfg.BoolOpt('libvirt_inject_key', + default=True, + help='Inject the ssh public key at boot time'), ++ cfg.IntOpt('libvirt_inject_partition', ++ default=1, ++ help='The partition to inject to : ' ++ '-1 => inspect (libguestfs only), 0 => not partitioned, ' ++ '>0 => partition number'), + cfg.BoolOpt('use_usb_tablet', + default=True, + help='Sync virtual and real mouse cursors in Windows VMs'), +@@ -1285,12 +1290,11 @@ class LibvirtConnection(driver.ComputeDriver): + cow=FLAGS.use_cow_images, + swap_mb=swap_mb) + +- # For now, we assume that if we're not using a kernel, we're using a +- # partitioned disk image where the target partition is the first +- # partition + target_partition = None + if not instance['kernel_id']: +- target_partition = "1" ++ target_partition = FLAGS.libvirt_inject_partition ++ if target_partition == 0: ++ target_partition = None + + config_drive_id = instance.get('config_drive_id') + config_drive = instance.get('config_drive') diff --git a/0028-handle-updated-qemu-img-info-output.patch b/0028-handle-updated-qemu-img-info-output.patch new file mode 100644 index 0000000..bb453f3 --- /dev/null +++ b/0028-handle-updated-qemu-img-info-output.patch @@ -0,0 +1,55 @@ +From 4099a82112d192ba01cb3c5fb3a71b5ef8bb7683 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?P=C3=A1draig=20Brady?= +Date: Wed, 16 May 2012 13:44:46 +0100 +Subject: [PATCH] handle updated qemu-img info output + +Originally `qemu-img info` always output an (actual path: ...) +even if it was duplicated with that already on the line. + + $ instances=/var/lib/nova/instances/ + $ qemu-img info $instances/instance-00000017/disk | grep 'backing' + backing file: $instances/_base/24083... (actual path: $the_same) + +Whereas after the change referenced at: +https://lists.gnu.org/archive/html/qemu-devel/2012-05/msg01468.html +It suppresses a duplicate (actual path:) + + $ instances=/var/lib/nova/instances/ + $ qemu-img info $instances/instance-00000017/disk | grep 'backing' + backing file: $instances/_base/24083... + +* nova/virt/libvirt/utils.py (get_disk_backing_file): +Avoid an indexError exception when parsing the newer format. +Fixes bug 1000261 + +Change-Id: Ie2889b6da8a5c93e0e874e7a330529f6e6e71b0b +--- + nova/virt/libvirt/utils.py | 14 +++++++++++--- + 1 files changed, 11 insertions(+), 3 deletions(-) + +diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py +index db7e11b..977eda8 100644 +--- a/nova/virt/libvirt/utils.py ++++ b/nova/virt/libvirt/utils.py +@@ -92,10 +92,18 @@ def get_disk_backing_file(path): + :returns: a path to the image's backing store + """ + out, err = execute('qemu-img', 'info', path) +- backing_file = [i.split('actual path:')[1].strip()[:-1] +- for i in out.split('\n') if 0 <= i.find('backing file')] ++ backing_file = None ++ ++ for line in out.split('\n'): ++ if line.startswith('backing file: '): ++ if 'actual path: ' in line: ++ backing_file = line.split('actual path: ')[1][:-1] ++ else: ++ backing_file = line.split('backing file: ')[1] ++ break + if backing_file: +- backing_file = os.path.basename(backing_file[0]) ++ backing_file = os.path.basename(backing_file) ++ + return backing_file + + diff --git a/openstack-nova.spec b/openstack-nova.spec index 8f1bff9..6817eaa 100644 --- a/openstack-nova.spec +++ b/openstack-nova.spec @@ -2,7 +2,7 @@ Name: openstack-nova Version: 2012.1 -Release: 4%{?dist} +Release: 5%{?dist} Summary: OpenStack Compute (nova) Group: Applications/System @@ -41,11 +41,26 @@ Patch0005: 0005-Fix-errors-in-os-networks-extension.patch Patch0006: 0006-Create-compute.api.BaseAPI-for-compute-APIs-to-use.patch Patch0007: 0007-Populate-image-properties-with-project_id-again.patch Patch0008: 0008-Use-project_id-in-ec2.cloud._format_image.patch -Patch0009: 0009-ensure-atomic-manipulation-of-libvirt-disk-images.patch -Patch0010: 0010-Ensure-we-don-t-access-the-net-when-building-docs.patch -Patch0011: 0011-fix-useexisting-deprecation-warnings.patch -Patch0012: 0012-support-a-configurable-libvirt-injection-partition.patch -Patch0013: 0013-enforce-quota-on-security-group-rules.patch +Patch0009: 0009-Implement-quotas-for-security-groups.patch +Patch0010: 0010-Delete-fixed_ips-when-network-is-deleted.patch +Patch0011: 0011-Xen-Pass-session-to-destroy_vdi.patch +Patch0012: 0012-add-libvirt_inject_key-flag.patch +Patch0013: 0013-Cloudpipe-tap-vpn-not-always-working.patch +Patch0014: 0014-Don-t-leak-RPC-connections-on-timeouts-or-other-exce.patch +Patch0015: 0015-Fixes-bug-987335.patch +Patch0016: 0016-Fix-timeout-in-EC2-CloudController.create_image.patch +Patch0017: 0017-Update-KillFilter-to-handle-deleted-exe-s.patch +Patch0018: 0018-Get-unit-tests-functional-in-OS-X.patch +Patch0019: 0019-Introduced-flag-base_dir_name.-Fixes-bug-973194.patch +Patch0020: 0020-Fix-bug-983206-_try_convert-parsing-string.patch +Patch0021: 0021-QuantumManager-will-start-dnsmasq-during-startup.-Fi.patch +Patch0022: 0022-Fixes-bug-952176.patch +Patch0023: 0023-Fix-nova.tests.test_nova_rootwrap-on-Fedora-17.patch +Patch0024: 0024-ensure-atomic-manipulation-of-libvirt-disk-images.patch +Patch0025: 0025-Ensure-we-don-t-access-the-net-when-building-docs.patch +Patch0026: 0026-fix-useexisting-deprecation-warnings.patch +Patch0027: 0027-support-a-configurable-libvirt-injection-partition.patch +Patch0028: 0028-handle-updated-qemu-img-info-output.patch BuildArch: noarch BuildRequires: intltool @@ -193,6 +208,21 @@ This package contains documentation files for nova. %patch0011 -p1 %patch0012 -p1 %patch0013 -p1 +%patch0014 -p1 +%patch0015 -p1 +%patch0016 -p1 +%patch0017 -p1 +%patch0018 -p1 +%patch0019 -p1 +%patch0020 -p1 +%patch0021 -p1 +%patch0022 -p1 +%patch0023 -p1 +%patch0024 -p1 +%patch0025 -p1 +%patch0026 -p1 +%patch0027 -p1 +%patch0028 -p1 find . \( -name .gitignore -o -name .placeholder \) -delete @@ -394,6 +424,10 @@ fi %endif %changelog +* Wed May 16 2012 Pádraig Brady - 2012.1-5 +- Sync up with Essex stable branch +- Handle updated qemu-img info output + * Wed May 09 2012 Alan Pevec - 2012.1-4 - Remove the socat dependency no longer needed by Essex