From fbcecf5737714206d43bb50af2cce88e074c462c Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 25 Nov 2011 20:31:06 -0500 Subject: [PATCH 01/55] git ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0205d62 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +.DS_Store From f265731fc91f800ec2a4d19d0f6b3192927d4605 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 25 Nov 2011 20:31:38 -0500 Subject: [PATCH 02/55] appengine key and ancestor support --- db/compiler.py | 81 +++++++++++++++++------- db/creation.py | 4 +- fields.py | 69 ++++++++++++++++++++ models.py | 66 +++++++++++++++++++ tests/__init__.py | 1 + tests/keys.py | 158 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 356 insertions(+), 23 deletions(-) create mode 100644 fields.py create mode 100644 tests/keys.py diff --git a/db/compiler.py b/db/compiler.py index f76fcfa..11c0f4f 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -1,6 +1,8 @@ from .db_settings import get_model_indexes from .utils import commit_locked from .expressions import ExpressionEvaluator +from ..fields import GAEKeyField +from ..models import GAEKey, GAEAncestorKey import datetime import sys @@ -73,6 +75,7 @@ def __init__(self, compiler, fields): self.pk_filters = None self.excluded_pks = () self.has_negated_exact_filter = False + self.ancestor_key = None self.ordering = () self.gae_ordering = [] pks_only = False @@ -173,6 +176,14 @@ def add_filter(self, column, lookup_type, negated, db_type, value): if column == self.query.get_meta().pk.column: column = '__key__' db_table = self.query.get_meta().db_table + + if lookup_type == 'exact' and isinstance(value, GAEAncestorKey): + if negated: + raise DatabaseError("You can't negate an ancestor operation.") + if self.ancestor_key is not None: + raise DatabaseError("You can't use more than one ancestor operation.") + self.ancestor_key = value.key() + return if lookup_type in ('exact', 'in'): # Optimization: batch-get by key if self.pk_filters is not None: @@ -319,6 +330,8 @@ def _make_entity(self, entity): def _build_query(self): for query in self.gae_query: query.Order(*self.gae_ordering) + if self.ancestor_key: + query.Ancestor(self.ancestor_key) if len(self.gae_query) > 1: return MultiQuery(self.gae_query, self.gae_ordering) return self.gae_query[0] @@ -391,24 +404,24 @@ def convert_value_from_db(self, db_type, value): # contain non unicode strings, nevertheless work with unicode ones) value = value.decode('utf-8') elif isinstance(value, Key): - # for now we do not support KeyFields thus a Key has to be the own - # primary key - # TODO: GAE: support parents via GAEKeyField - assert value.parent() is None, "Parents are not yet supported!" - if db_type == 'integer': - if value.id() is None: - raise DatabaseError('Wrong type for Key. Expected integer, found' - 'None') - else: - value = value.id() - elif db_type == 'text': - if value.name() is None: - raise DatabaseError('Wrong type for Key. Expected string, found' - 'None') - else: - value = value.name() + if db_type == 'gae_key': + value = GAEKey(real_key=value) else: - raise DatabaseError("%s fields cannot be keys on GAE" % db_type) + assert value.parent() is None, "Use GAEKeyField to enable parent keys!" + if db_type == 'integer': + if value.id() is None: + raise DatabaseError('Wrong type for Key. Expected integer, found' + 'None') + else: + value = value.id() + elif db_type == 'text': + if value.name() is None: + raise DatabaseError('Wrong type for Key. Expected string, found' + 'None') + else: + value = value.name() + else: + raise DatabaseError("%s fields cannot be keys on GAE" % db_type) elif db_type == 'date' and isinstance(value, datetime.datetime): value = value.date() elif db_type == 'time' and isinstance(value, datetime.datetime): @@ -435,7 +448,10 @@ def convert_value_for_db(self, db_type, value): value = Blob(pickle.dumps(value)) if db_type == 'gae_key': - return value + if isinstance(value, GAEKey) and value.has_real_key(): + return value.real_key + else: + return value elif db_type == 'longtext': # long text fields cannot be indexed on GAE so use GAE's database # type Text @@ -465,21 +481,37 @@ def insert(self, data, return_id=False): kwds = {'unindexed_properties': unindexed_cols} for column, value in data.items(): if column == opts.pk.column: - if isinstance(value, basestring): + if isinstance(value, GAEKey): + if value.parent_key and value.parent_key.has_real_key(): + kwds['parent'] = value.parent_key.real_key + if isinstance(value.id_or_name, basestring): + kwds['name'] = value.id_or_name + elif value.id_or_name is not None: + kwds['id'] = value.id_or_name + elif isinstance(value, Key): + kwds['parent'] = value.parent() + if value.name(): + kwds['name'] = value.name() + elif value.id(): + kwds['id'] = value.id() + elif isinstance(value, basestring): kwds['name'] = value else: kwds['id'] = value elif isinstance(value, (tuple, list)) and not len(value): - # gae does not store emty lists (and even does not allow passing empty + # gae does not store empty lists (and even does not allow passing empty # lists to Entity.update) so skip them continue else: gae_data[column] = value - entity = Entity(self.query.get_meta().db_table, **kwds) + entity = Entity(opts.db_table, **kwds) entity.update(gae_data) key = Put(entity) - return key.id_or_name() + + if not isinstance(opts.pk, GAEKeyField): + key = key.id_or_name() + return key class SQLUpdateCompiler(NonrelUpdateCompiler, SQLCompiler): def execute_sql(self, result_type=MULTI): @@ -556,6 +588,11 @@ def to_datetime(value): value.second, value.microsecond) def create_key(db_table, value): + if isinstance(value, GAEKey): + parent = None + if value.parent_key is not None: + parent = value.parent.real_key + return Key.from_path(db_table, value.id_or_name, parent=parent) if isinstance(value, (int, long)) and value < 1: return None return Key.from_path(db_table, value) diff --git a/db/creation.py b/db/creation.py index 912a52e..42f4f0f 100644 --- a/db/creation.py +++ b/db/creation.py @@ -15,12 +15,14 @@ def __mod__(self, field): return self.internal_type def get_data_types(): - # TODO: Add GAEKeyField and a corresponding db_type string_types = ('text', 'longtext') data_types = NonrelDatabaseCreation.data_types.copy() for name, field_type in data_types.items(): if field_type in string_types: data_types[name] = StringType(field_type) + + data_types['GAEKeyField'] = 'gae_key' + return data_types class DatabaseCreation(NonrelDatabaseCreation): diff --git a/fields.py b/fields.py new file mode 100644 index 0000000..2131a8d --- /dev/null +++ b/fields.py @@ -0,0 +1,69 @@ +from django.db import models +from google.appengine.api.datastore import Key +from .models import GAEKey, GAEAncestorKey + +class GAEKeyField(models.Field): + description = "A field for Google AppEngine Key objects" + __metaclass__ = models.SubfieldBase + + def __init__(self, *args, **kwargs): + assert kwargs.get('primary_key', False) is True, "%ss must have primary_key=True." % self.__class__.__name__ + kwargs['null'] = True + kwargs['blank'] = True + self.parent_key_attname = kwargs.pop('parent_key_name', None) + + super(GAEKeyField, self).__init__(*args, **kwargs) + + def contribute_to_class(self, cls, name): + assert not cls._meta.has_auto_field, "A model can't have more than one auto field." + super(GAEKeyField, self).contribute_to_class(cls, name) + cls._meta.has_auto_field = True + cls._meta.auto_field = self + + if self.parent_key_attname is not None: + def get_parent_key(instance, instance_type=None): + if instance is None: + return self + return instance.__dict__.get(self.parent_key_attname) + + def set_parent_key(instance, value): + if instance is None: + raise AttributeError("Attribute must be accessed via instance") + + if not isinstance(value, GAEKey): + raise ValueError("parent must be a GAEKey") + + instance.__dict__[self.parent_key_attname] = value + + setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) + + def to_python(self, value): + if value is None: + return None + if isinstance(value, GAEKey): + return value + if isinstance(value, Key): + return GAEKey(real_key=value) + if isinstance(value, basestring): + return GAEKey(real_key=Key(encoded=value)) + return GAEKey(id_or_name=value) + + def get_prep_value(self, value): + if value is None: + return None + if not isinstance(value, (GAEKey, GAEAncestorKey)): + raise ValueError('must by type GAEKey or GAEAncestorKey, not <%s>' % type(value)) + return value + + def formfield(self, **kwargs): + return None + + def pre_save(self, model_instance, add): + if add and self.parent_key_attname is not None: + parent_key = getattr(model_instance, self.parent_key_attname) + if parent_key is not None: + key = GAEKey(parent_key=parent_key) + setattr(model_instance, self.attname, key) + return key + + return super(GAEKeyField, self).pre_save(model_instance, add) diff --git a/models.py b/models.py index e69de29..788445b 100644 --- a/models.py +++ b/models.py @@ -0,0 +1,66 @@ +from django.db import models +from google.appengine.api.datastore import Key + +# TODO: look for better exceptions to raise + +class GAEAncestorKey(object): + def __init__(self, key): + if not isinstance(key, Key): + raise ValueError('key must be of type Key') + + self._key = key + + def key(self): + return self._key + +class GAEKey(object): + def __init__(self, id_or_name=None, parent_key=None, real_key=None): + self._id_or_name = id_or_name + self._parent_key = parent_key + self._real_key = None + + if real_key is not None: + if id_or_name is not None or parent_key is not None: + raise ValueError("You can't set both a real_key and an id_or_name or parent_key") + + self._real_key = real_key + if real_key.parent(): + self._parent_key = GAEKey(real_key=real_key.parent()) + self._id_or_name = real_key.id_or_name() + + def _get_id_or_name(self): + return self._id_or_name + id_or_name = property(_get_id_or_name) + + def _get_parent_key(self): + return self._parent_key + parent_key = property(_get_parent_key) + + def _get_real_key(self): + if self._real_key is None: + raise ValueError("Incomplete key, please save the entity first.") + return self._real_key + real_key = property(_get_real_key) + + def has_real_key(self): + return self._real_key is not None + + def as_ancestor(self): + return GAEAncestorKey(self._get_real_key()) + + def __cmp__(self, other): + if not isinstance(other, GAEKey): + return 1 + if self._real_key is None or other._real_key is None: + raise ValueError("You can't compare unsaved keys.") + + return cmp(self._real_key, other._real_key) + + def __hash__(self): + if self._real_key is None: + raise ValueError("You can't hash an unsaved key.") + + return hash(self._real_key) + + def __str__(self): + return str(self._real_key) diff --git a/tests/__init__.py b/tests/__init__.py index c5f867d..cb4e8ab 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,3 +6,4 @@ from .not_return_sets import NonReturnSetsTest from .decimals import DecimalTest from .transactions import TransactionTest +from .keys import KeysTest diff --git a/tests/keys.py b/tests/keys.py new file mode 100644 index 0000000..e852393 --- /dev/null +++ b/tests/keys.py @@ -0,0 +1,158 @@ +from django.db import models +from django.test import TestCase +from django.db.utils import DatabaseError + +from google.appengine.api.datastore import Key + +from ..fields import GAEKeyField +from ..models import GAEKey + +class ParentModel(models.Model): + key = GAEKeyField(primary_key=True) + +class NonGAEParentModel(models.Model): + id = models.AutoField(primary_key=True) + +class ChildModel(models.Model): + key = GAEKeyField(primary_key=True, parent_key_name='parent_key') + +class AnotherChildModel(models.Model): + key = GAEKeyField(primary_key=True, parent_key_name='also_parent_key') + +class ForeignKeyModel(models.Model): + id = models.AutoField(primary_key=True) + relation = models.ForeignKey(ParentModel) + +class KeysTest(TestCase): + def testGAEKeySave(self): + model = ParentModel() + model.save() + + self.assertIsNotNone(model.pk) + + def testUnsavedParent(self): + parent = ParentModel() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent.pk) + + def testNonGAEParent(self): + parent = NonGAEParentModel() + parent.save() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent.pk) + + def testParentChildSave(self): + parent = ParentModel() + orig_parent_pk = parent.pk + parent.save() + + child = ChildModel(parent_key=parent.pk) + orig_child_pk = child.pk + child.save() + + self.assertNotEquals(parent.pk, orig_parent_pk) + self.assertNotEquals(child.pk, orig_child_pk) + self.assertEquals(child.pk.parent_key, parent.pk) + self.assertEquals(child.pk.parent_key.real_key, parent.pk.real_key) + + def testAncestorFilterQuery(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + child.save() + + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) + + def testAncestorGetQuery(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + child.save() + + result = ChildModel.objects.get(pk=parent.pk.as_ancestor()) + + self.assertEquals(result.pk, child.pk) + + def testEmptyAncestorQuery(self): + parent = ParentModel() + parent.save() + + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + self.assertEquals(0, len(results)) + + def testEmptyAncestorQueryWithUnsavedChild(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + self.assertEquals(0, len(results)) + + def testUnsavedAncestorQuery(self): + parent = ParentModel() + + with self.assertRaises(AttributeError): + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + def testDifferentChildrenAncestorQuery(self): + parent = ParentModel() + parent.save() + + child1 = ChildModel(parent_key=parent.pk) + child1.save() + child2 = AnotherChildModel(also_parent_key=parent.pk) + child2.save() + + results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child1.pk) + + results = list(AnotherChildModel.objects.filter(pk=parent.pk.as_ancestor())) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child2.pk) + + def testDifferentParentsAncestorQuery(self): + parent1 = ParentModel() + parent1.save() + + child1 = ChildModel(parent_key=parent1.pk) + child1.save() + + parent2 = ParentModel() + parent2.save() + + child2 = ChildModel(parent_key=parent2.pk) + child2.save() + + results = list(ChildModel.objects.filter(pk=parent1.pk.as_ancestor())) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child1.pk) + + results = list(ChildModel.objects.filter(pk=parent2.pk.as_ancestor())) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child2.pk) + + def testForeignKeyWithGAEKey(self): + parent = ParentModel() + parent.save() + + fkm = ForeignKeyModel() + fkm.relation = parent + fkm.save() + + results = list(ForeignKeyModel.objects.filter(relation=parent)) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, fkm.pk) + From decfc123debbb6376016adeecfe1d9eda1d6806c Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 25 Nov 2011 20:37:11 -0500 Subject: [PATCH 03/55] fixes for non-primary GAEKeys --- db/compiler.py | 3 ++- fields.py | 56 +++++++++++++++++++++++++++++++------------------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/db/compiler.py b/db/compiler.py index 11c0f4f..bdc7a3c 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -511,6 +511,7 @@ def insert(self, data, return_id=False): if not isinstance(opts.pk, GAEKeyField): key = key.id_or_name() + return key class SQLUpdateCompiler(NonrelUpdateCompiler, SQLCompiler): @@ -591,7 +592,7 @@ def create_key(db_table, value): if isinstance(value, GAEKey): parent = None if value.parent_key is not None: - parent = value.parent.real_key + parent = value.parent_key.real_key return Key.from_path(db_table, value.id_or_name, parent=parent) if isinstance(value, (int, long)) and value < 1: return None diff --git a/fields.py b/fields.py index 2131a8d..1408777 100644 --- a/fields.py +++ b/fields.py @@ -1,5 +1,5 @@ from django.db import models -from google.appengine.api.datastore import Key +from google.appengine.api.datastore import Key, datastore_errors from .models import GAEKey, GAEAncestorKey class GAEKeyField(models.Field): @@ -7,35 +7,39 @@ class GAEKeyField(models.Field): __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): - assert kwargs.get('primary_key', False) is True, "%ss must have primary_key=True." % self.__class__.__name__ kwargs['null'] = True kwargs['blank'] = True self.parent_key_attname = kwargs.pop('parent_key_name', None) + if self.parent_key_attname is not None and kwargs.get('primary_key', None) is None: + raise ValueError("Primary key must be true to set parent_key_name") + super(GAEKeyField, self).__init__(*args, **kwargs) def contribute_to_class(self, cls, name): - assert not cls._meta.has_auto_field, "A model can't have more than one auto field." - super(GAEKeyField, self).contribute_to_class(cls, name) - cls._meta.has_auto_field = True - cls._meta.auto_field = self + if self.primary_key: + assert not cls._meta.has_auto_field, "A model can't have more than one auto field." + cls._meta.has_auto_field = True + cls._meta.auto_field = self - if self.parent_key_attname is not None: - def get_parent_key(instance, instance_type=None): - if instance is None: - return self - return instance.__dict__.get(self.parent_key_attname) + if self.parent_key_attname is not None: + def get_parent_key(instance, instance_type=None): + if instance is None: + return self + return instance.__dict__.get(self.parent_key_attname) - def set_parent_key(instance, value): - if instance is None: - raise AttributeError("Attribute must be accessed via instance") + def set_parent_key(instance, value): + if instance is None: + raise AttributeError("Attribute must be accessed via instance") - if not isinstance(value, GAEKey): - raise ValueError("parent must be a GAEKey") + if not isinstance(value, GAEKey): + raise ValueError("parent must be a GAEKey") - instance.__dict__[self.parent_key_attname] = value + instance.__dict__[self.parent_key_attname] = value - setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) + setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) + + super(GAEKeyField, self).contribute_to_class(cls, name) def to_python(self, value): if value is None: @@ -45,14 +49,24 @@ def to_python(self, value): if isinstance(value, Key): return GAEKey(real_key=value) if isinstance(value, basestring): - return GAEKey(real_key=Key(encoded=value)) - return GAEKey(id_or_name=value) + try: + return GAEKey(real_key=Key(encoded=value)) + except datastore_errors.BadKeyError: + pass + raise ValueError("this value is not allowed %s" % value) def get_prep_value(self, value): if value is None: return None + if isinstance(value, Key): + return GAEKey(real_key=value) + if isinstance(value, basestring): + try: + return GAEKey(real_key=Key(encoded=value)) + except datastore_errors.BadKeyError: + raise ValueError("this value is not allowed %s" % value) if not isinstance(value, (GAEKey, GAEAncestorKey)): - raise ValueError('must by type GAEKey or GAEAncestorKey, not <%s>' % type(value)) + raise ValueError('Must by type GAEKey, GAEAncestorKey, basestring. Not <%s>' % type(value)) return value def formfield(self, **kwargs): From 87a9f855d94b5b9dcd3e12bcd1068641b079bc1e Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Mon, 28 Nov 2011 22:56:16 -0500 Subject: [PATCH 04/55] use id_or_name as string value --- models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models.py b/models.py index 788445b..91a00fd 100644 --- a/models.py +++ b/models.py @@ -38,7 +38,7 @@ def _get_parent_key(self): def _get_real_key(self): if self._real_key is None: - raise ValueError("Incomplete key, please save the entity first.") + raise AttributeError("Incomplete key, please save the entity first.") return self._real_key real_key = property(_get_real_key) @@ -63,4 +63,4 @@ def __hash__(self): return hash(self._real_key) def __str__(self): - return str(self._real_key) + return str(self.id_or_name) From fd234e61812b45415fc4380ba5030498195f02c6 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Mon, 28 Nov 2011 22:57:26 -0500 Subject: [PATCH 05/55] better value conversion to match str from GAEKey --- fields.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/fields.py b/fields.py index 1408777..8c711e4 100644 --- a/fields.py +++ b/fields.py @@ -1,3 +1,4 @@ +from django.core.exceptions import ValidationError from django.db import models from google.appengine.api.datastore import Key, datastore_errors from .models import GAEKey, GAEAncestorKey @@ -52,22 +53,16 @@ def to_python(self, value): try: return GAEKey(real_key=Key(encoded=value)) except datastore_errors.BadKeyError: - pass - raise ValueError("this value is not allowed %s" % value) + return GAEKey(real_key=Key.from_path(self.model._meta.db_table, long(value))) + if isinstance(value, (int, long)): + return GAEKey(real_key=Key.from_path(self.model._meta.db_table, value)) + + raise ValidationError("GAEKeyField does not accept %s" % type(value)) def get_prep_value(self, value): - if value is None: - return None - if isinstance(value, Key): - return GAEKey(real_key=value) - if isinstance(value, basestring): - try: - return GAEKey(real_key=Key(encoded=value)) - except datastore_errors.BadKeyError: - raise ValueError("this value is not allowed %s" % value) - if not isinstance(value, (GAEKey, GAEAncestorKey)): - raise ValueError('Must by type GAEKey, GAEAncestorKey, basestring. Not <%s>' % type(value)) - return value + if isinstance(value, GAEAncestorKey): + return value + return self.to_python(value) def formfield(self, **kwargs): return None From 150c915a4defdae2ba90f2581bda9d2e63aeb424 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Mon, 28 Nov 2011 22:57:48 -0500 Subject: [PATCH 06/55] test cases for different kinds of pk representations --- tests/keys.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/keys.py b/tests/keys.py index e852393..4e6bd06 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -147,12 +147,36 @@ def testDifferentParentsAncestorQuery(self): def testForeignKeyWithGAEKey(self): parent = ParentModel() parent.save() - + fkm = ForeignKeyModel() fkm.relation = parent fkm.save() - + results = list(ForeignKeyModel.objects.filter(relation=parent)) self.assertEquals(1, len(results)) self.assertEquals(results[0].pk, fkm.pk) + def testPrimaryKeyQuery(self): + parent = ParentModel() + parent.save() + + db_parent = ParentModel.objects.get(pk=parent.pk) + + self.assertEquals(parent.pk, db_parent.pk) + + def testPrimaryKeyQueryStringKey(self): + parent = ParentModel() + parent.save() + + db_parent = ParentModel.objects.get(pk=str(parent.pk)) + + self.assertEquals(parent.pk, db_parent.pk) + + def testPrimaryKeyQueryIntKey(self): + parent = ParentModel() + parent.save() + + db_parent = ParentModel.objects.get(pk=int(str(parent.pk))) + + self.assertEquals(parent.pk, db_parent.pk) + \ No newline at end of file From 29a1c620b7305678f415ad4ce87eb1ea3c41f268 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Tue, 6 Dec 2011 11:31:36 -0500 Subject: [PATCH 07/55] fix cmp for GAEKey --- models.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/models.py b/models.py index 91a00fd..f0fb719 100644 --- a/models.py +++ b/models.py @@ -1,8 +1,6 @@ from django.db import models from google.appengine.api.datastore import Key -# TODO: look for better exceptions to raise - class GAEAncestorKey(object): def __init__(self, key): if not isinstance(key, Key): @@ -51,10 +49,21 @@ def as_ancestor(self): def __cmp__(self, other): if not isinstance(other, GAEKey): return 1 - if self._real_key is None or other._real_key is None: - raise ValueError("You can't compare unsaved keys.") - - return cmp(self._real_key, other._real_key) + + if self._real_key is not None and other._real_key is not None: + return cmp(self._real_key, other._real_key) + + if self._id_or_name is None or other._id_or_name is None: + raise ValueError("You can't compare unsaved keys: %s %s" % (self, other)) + + result = 0 + if self._parent_key is not None: + result = cmp(self._parent_key, other._parent_key) + + if result == 0: + result = cmp(self._id_or_name, other._id_or_name) + + return result def __hash__(self): if self._real_key is None: From 091b12ca010794e194e5959ceb3694458f00819f Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Thu, 29 Dec 2011 13:22:54 -0500 Subject: [PATCH 08/55] use methods instead of properties for GAEKey fields --- db/compiler.py | 20 ++++++++++---------- models.py | 14 ++++++-------- tests/keys.py | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/db/compiler.py b/db/compiler.py index bdc7a3c..5c0934c 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -449,7 +449,7 @@ def convert_value_for_db(self, db_type, value): if db_type == 'gae_key': if isinstance(value, GAEKey) and value.has_real_key(): - return value.real_key + return value.real_key() else: return value elif db_type == 'longtext': @@ -482,12 +482,12 @@ def insert(self, data, return_id=False): for column, value in data.items(): if column == opts.pk.column: if isinstance(value, GAEKey): - if value.parent_key and value.parent_key.has_real_key(): - kwds['parent'] = value.parent_key.real_key - if isinstance(value.id_or_name, basestring): - kwds['name'] = value.id_or_name - elif value.id_or_name is not None: - kwds['id'] = value.id_or_name + if value.parent_key() and value.parent_key().has_real_key(): + kwds['parent'] = value.parent_key().real_key() + if isinstance(value.id_or_name(), basestring): + kwds['name'] = value.id_or_name() + elif value.id_or_name() is not None: + kwds['id'] = value.id_or_name() elif isinstance(value, Key): kwds['parent'] = value.parent() if value.name(): @@ -591,9 +591,9 @@ def to_datetime(value): def create_key(db_table, value): if isinstance(value, GAEKey): parent = None - if value.parent_key is not None: - parent = value.parent_key.real_key - return Key.from_path(db_table, value.id_or_name, parent=parent) + if value.parent_key() is not None: + parent = value.parent_key().real_key() + return Key.from_path(db_table, value.id_or_name(), parent=parent) if isinstance(value, (int, long)) and value < 1: return None return Key.from_path(db_table, value) diff --git a/models.py b/models.py index f0fb719..b3852e9 100644 --- a/models.py +++ b/models.py @@ -8,6 +8,7 @@ def __init__(self, key): self._key = key + @property def key(self): return self._key @@ -26,25 +27,22 @@ def __init__(self, id_or_name=None, parent_key=None, real_key=None): self._parent_key = GAEKey(real_key=real_key.parent()) self._id_or_name = real_key.id_or_name() - def _get_id_or_name(self): + def id_or_name(self): return self._id_or_name - id_or_name = property(_get_id_or_name) - def _get_parent_key(self): + def parent_key(self): return self._parent_key - parent_key = property(_get_parent_key) - def _get_real_key(self): + def real_key(self): if self._real_key is None: raise AttributeError("Incomplete key, please save the entity first.") return self._real_key - real_key = property(_get_real_key) def has_real_key(self): return self._real_key is not None def as_ancestor(self): - return GAEAncestorKey(self._get_real_key()) + return GAEAncestorKey(self.real_key()) def __cmp__(self, other): if not isinstance(other, GAEKey): @@ -72,4 +70,4 @@ def __hash__(self): return hash(self._real_key) def __str__(self): - return str(self.id_or_name) + return str(self._id_or_name) diff --git a/tests/keys.py b/tests/keys.py index 4e6bd06..f6be381 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -55,7 +55,7 @@ def testParentChildSave(self): self.assertNotEquals(parent.pk, orig_parent_pk) self.assertNotEquals(child.pk, orig_child_pk) self.assertEquals(child.pk.parent_key, parent.pk) - self.assertEquals(child.pk.parent_key.real_key, parent.pk.real_key) + self.assertEquals(child.pk.parent_key.real_key(), parent.pk.real_key()) def testAncestorFilterQuery(self): parent = ParentModel() From 902c3638a23f275a3e665315797b03e87ac55854 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Thu, 29 Dec 2011 16:18:50 -0500 Subject: [PATCH 09/55] remove missed @property, update unit tests --- models.py | 1 - tests/keys.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/models.py b/models.py index b3852e9..32a9da7 100644 --- a/models.py +++ b/models.py @@ -8,7 +8,6 @@ def __init__(self, key): self._key = key - @property def key(self): return self._key diff --git a/tests/keys.py b/tests/keys.py index f6be381..01b76f8 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -54,8 +54,8 @@ def testParentChildSave(self): self.assertNotEquals(parent.pk, orig_parent_pk) self.assertNotEquals(child.pk, orig_child_pk) - self.assertEquals(child.pk.parent_key, parent.pk) - self.assertEquals(child.pk.parent_key.real_key(), parent.pk.real_key()) + self.assertEquals(child.pk.parent_key(), parent.pk) + self.assertEquals(child.pk.parent_key().real_key(), parent.pk.real_key()) def testAncestorFilterQuery(self): parent = ParentModel() From 6aa66b7857e1ca71645142486512ade270335a6d Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Wed, 25 Jan 2012 16:53:30 -0500 Subject: [PATCH 10/55] use real_key for serialization --- fields.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fields.py b/fields.py index 8c711e4..f81691c 100644 --- a/fields.py +++ b/fields.py @@ -1,5 +1,6 @@ from django.core.exceptions import ValidationError from django.db import models +from django.utils.encoding import smart_unicode from google.appengine.api.datastore import Key, datastore_errors from .models import GAEKey, GAEAncestorKey @@ -76,3 +77,6 @@ def pre_save(self, model_instance, add): return key return super(GAEKeyField, self).pre_save(model_instance, add) + + def value_to_string(self, obj): + return smart_unicode(self._get_val_from_obj(obj).real_key()) From 216aa48c7686cc5bc07254b5834f09def1f30c77 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 18:11:58 -0400 Subject: [PATCH 11/55] modify SqlInsertCompiler to handle bulk inserts --- db/compiler.py | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/db/compiler.py b/db/compiler.py index 7d9a249..5bd03a5 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -453,30 +453,36 @@ def convert_value_for_db(self, db_type, value): class SQLInsertCompiler(NonrelInsertCompiler, SQLCompiler): @safe_call - def insert(self, data, return_id=False): - gae_data = {} + def insert(self, docs, return_id=False): opts = self.query.get_meta() unindexed_fields = get_model_indexes(self.query.model)['unindexed'] unindexed_cols = [opts.get_field(name).column for name in unindexed_fields] - kwds = {'unindexed_properties': unindexed_cols} - for column, value in data.items(): - if column == opts.pk.column: - if isinstance(value, basestring): - kwds['name'] = value - else: - kwds['id'] = value - elif isinstance(value, (tuple, list)) and not len(value): - # gae does not store emty lists (and even does not allow passing empty - # lists to Entity.update) so skip them - continue - else: - gae_data[column] = value - entity = Entity(self.query.get_meta().db_table, **kwds) - entity.update(gae_data) - key = Put(entity) - return key.id_or_name() + entity_list = [] + for data in docs: + gae_data = {} + kwds = {'unindexed_properties': unindexed_cols} + for column, value in data.items(): + if column == opts.pk.column: + if isinstance(value, basestring): + kwds['name'] = value + else: + kwds['id'] = value + elif isinstance(value, (tuple, list)) and not len(value): + # gae does not store emty lists (and even does not allow passing empty + # lists to Entity.update) so skip them + continue + else: + gae_data[column] = value + + entity = Entity(opts.db_table, **kwds) + entity.update(gae_data) + entity_list.append(entity) + keys = Put(entity_list) + if not isinstance(keys, list): + keys = [keys] + return keys[0].id_or_name() class SQLUpdateCompiler(NonrelUpdateCompiler, SQLCompiler): def execute_sql(self, result_type=MULTI): From 7df916edbf39a99309a1b2ac6aaa95c5fd871530 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 18:12:07 -0400 Subject: [PATCH 12/55] remove usage of XMLField --- tests/field_db_conversion.py | 6 +++--- tests/testmodels.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/field_db_conversion.py b/tests/field_db_conversion.py index e8a50d5..882322a 100644 --- a/tests/field_db_conversion.py +++ b/tests/field_db_conversion.py @@ -18,7 +18,7 @@ def test_db_conversion(self): comma_seperated_integer="5,4,3,2", ip_address='194.167.1.1', slug='you slugy slut :)', url='http://www.scholardocs.com', long_text=1000*'A', - indexed_text='hello', xml=2000*'B', + indexed_text='hello', integer=-400, small_integer=-4, positiv_integer=400, positiv_small_integer=4) entity.save() @@ -29,7 +29,7 @@ def test_db_conversion(self): entity.pk)) for name, gae_db_type in [('long_text', Text), - ('indexed_text', unicode), ('xml', Text), + ('indexed_text', unicode), ('text', unicode), ('ip_address', unicode), ('slug', unicode), ('email', unicode), ('comma_seperated_integer', unicode), ('url', unicode), ('time', datetime.datetime), @@ -47,7 +47,7 @@ def test_db_conversion(self): # right types entity = FieldsWithoutOptionsModel.objects.get() for name, expected_type in [('long_text', unicode), - ('indexed_text', unicode), ('xml', unicode), + ('indexed_text', unicode), ('text', unicode), ('ip_address', unicode), ('slug', unicode), ('email', unicode), ('comma_seperated_integer', unicode), ('url', unicode), ('datetime', datetime.datetime), diff --git a/tests/testmodels.py b/tests/testmodels.py index 125a30f..4999969 100644 --- a/tests/testmodels.py +++ b/tests/testmodels.py @@ -28,7 +28,6 @@ class FieldsWithoutOptionsModel(models.Model): # file_path = models.FilePathField() long_text = models.TextField() indexed_text = models.TextField() - xml = models.XMLField() integer = models.IntegerField() small_integer = models.SmallIntegerField() positiv_integer = models.PositiveIntegerField() @@ -63,7 +62,6 @@ class FieldsWithOptionsModel(models.Model): # file = FileField() # file_path = FilePathField() long_text = models.TextField(default=1000*'A') - xml = models.XMLField(default=2000*'B') integer = models.IntegerField(default=100) small_integer = models.SmallIntegerField(default=-5) positiv_integer = models.PositiveIntegerField(default=80) From 82080679a2309b7b1c4214b0af7aa44e750fc220 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 19:29:02 -0400 Subject: [PATCH 13/55] repr method for GAEKey --- models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/models.py b/models.py index 32a9da7..b1fe7e2 100644 --- a/models.py +++ b/models.py @@ -46,20 +46,20 @@ def as_ancestor(self): def __cmp__(self, other): if not isinstance(other, GAEKey): return 1 - + if self._real_key is not None and other._real_key is not None: return cmp(self._real_key, other._real_key) - + if self._id_or_name is None or other._id_or_name is None: raise ValueError("You can't compare unsaved keys: %s %s" % (self, other)) result = 0 if self._parent_key is not None: result = cmp(self._parent_key, other._parent_key) - + if result == 0: result = cmp(self._id_or_name, other._id_or_name) - + return result def __hash__(self): @@ -70,3 +70,6 @@ def __hash__(self): def __str__(self): return str(self._id_or_name) + + def __repr__(self): + return "%s(id_or_name=%r, parent_key=%r, real_key=%r)" % (self.__class__, self._id_or_name, self._parent_key, self._real_key) From 2fc14a5cfe2d8ea1c732337e07f9580e4b50eb1c Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 19:32:03 -0400 Subject: [PATCH 14/55] do not encode cursor if it doesnt exist --- db/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/utils.py b/db/utils.py index 76085c3..5376d5c 100644 --- a/db/utils.py +++ b/db/utils.py @@ -16,7 +16,9 @@ def get_cursor(queryset): # Evaluate QuerySet len(queryset) cursor = getattr(queryset.query, '_gae_cursor', None) - return Cursor.to_websafe_string(cursor) + if cursor: + return Cursor.to_websafe_string(cursor) + return None def set_cursor(queryset, start=None, end=None): queryset = queryset.all() From b23cc69028f529bac25266e3f292d01d5a609072 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 17 Mar 2012 22:01:58 -0400 Subject: [PATCH 15/55] do not allow null primary keys, check for empty strings when creating keys --- fields.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fields.py b/fields.py index f81691c..de981ed 100644 --- a/fields.py +++ b/fields.py @@ -9,7 +9,6 @@ class GAEKeyField(models.Field): __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): - kwargs['null'] = True kwargs['blank'] = True self.parent_key_attname = kwargs.pop('parent_key_name', None) @@ -23,7 +22,7 @@ def contribute_to_class(self, cls, name): assert not cls._meta.has_auto_field, "A model can't have more than one auto field." cls._meta.has_auto_field = True cls._meta.auto_field = self - + if self.parent_key_attname is not None: def get_parent_key(instance, instance_type=None): if instance is None: @@ -46,6 +45,8 @@ def set_parent_key(instance, value): def to_python(self, value): if value is None: return None + if isinstance(value, basestring) and len(value) == 0: + return None if isinstance(value, GAEKey): return value if isinstance(value, Key): @@ -62,7 +63,7 @@ def to_python(self, value): def get_prep_value(self, value): if isinstance(value, GAEAncestorKey): - return value + return value return self.to_python(value) def formfield(self, **kwargs): From 6a10a9d3dcdb47db7e7808e2eecd26e82b02df2d Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 23 Mar 2012 15:50:40 -0400 Subject: [PATCH 16/55] rewrite ancestor queries for type-conversion-refactor --- db/base.py | 3 +- db/compiler.py | 11 ++-- db/utils.py | 18 +++++-- fields.py | 67 +++++++++++++----------- models.py | 75 -------------------------- tests/__init__.py | 4 +- tests/keys.py | 131 +++++++++++++++++++++------------------------- 7 files changed, 118 insertions(+), 191 deletions(-) diff --git a/db/base.py b/db/base.py index 24d04af..631bf36 100644 --- a/db/base.py +++ b/db/base.py @@ -155,7 +155,8 @@ def _value_for_db(self, value, field, field_kind, db_type, lookup): if db_type == 'key': # value = self._value_for_db_key(value, field_kind) try: - value = key_from_path(field.model._meta.db_table, value) + if not isinstance(value, Key): + value = key_from_path(field.model._meta.db_table, value) except (BadArgumentError, BadValueError,): raise DatabaseError("Only strings and positive integers " "may be used as keys on GAE.") diff --git a/db/compiler.py b/db/compiler.py index c880f4d..5c7eb5d 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -22,8 +22,6 @@ from .db_settings import get_model_indexes from .expressions import ExpressionEvaluator -from ..fields import GAEKeyField -from ..models import GAEKey, GAEAncestorKey from .utils import commit_locked @@ -193,12 +191,12 @@ def add_filter(self, field, lookup_type, negated, value): # Optimization: batch-get by key; this is only suitable for # primary keys, not for anything that uses the key type. if field.primary_key and lookup_type in ('exact', 'in'): - if lookup_type == 'exact' and isinstance(value, GAEAncestorKey): + if lookup_type == 'exact' and isinstance(value, AncestorKey): if negated: - raise DatabaseError("You can't negate an ancestor operation.") + raise DatabaseError("You can't negate an ancestor operator.") if self.ancestor_key is not None: - raise DatabaseError("You can't use more than one ancestor operation.") - self.ancestor_key = value.key() + raise DatabaseError("You can't use more than one ancestor operator.") + self.ancestor_key = value.key return if self.included_pks is not None: @@ -383,6 +381,7 @@ def insert(self, data, return_id=False): if value is not None: kwds['id'] = value.id() kwds['name'] = value.name() + kwds['parent'] = value.parent() # GAE does not store empty lists (and even does not allow # passing empty lists to Entity.update) so skip them. diff --git a/db/utils.py b/db/utils.py index 068715d..e324697 100644 --- a/db/utils.py +++ b/db/utils.py @@ -1,4 +1,6 @@ +from google.appengine.api.datastore import Key from google.appengine.datastore.datastore_query import Cursor + from django.db import models, DEFAULT_DB_ALIAS try: @@ -20,9 +22,7 @@ def get_cursor(queryset): # Evaluate QuerySet. len(queryset) cursor = getattr(queryset.query, '_gae_cursor', None) - if cursor: - return Cursor.to_websafe_string(cursor) - return None + return Cursor.to_websafe_string(cursor) if cursor else None def set_cursor(queryset, start=None, end=None): @@ -59,3 +59,15 @@ def _commit_locked(*args, **kw): if callable(func_or_using): return inner_commit_locked(func_or_using, DEFAULT_DB_ALIAS) return lambda func: inner_commit_locked(func, func_or_using) + +class AncestorKey(object): + def __init__(self, key): + self.key = key + +def as_ancestor(key): + if key is None: + raise ValueError("key must not be None") + return AncestorKey(key) + +def make_key(model, id_or_name, parent=None): + return Key.from_path(model._meta.db_table, id_or_name, parent=parent) diff --git a/fields.py b/fields.py index de981ed..e013482 100644 --- a/fields.py +++ b/fields.py @@ -1,21 +1,27 @@ from django.core.exceptions import ValidationError from django.db import models from django.utils.encoding import smart_unicode + +from djangoappengine.db.utils import AncestorKey + from google.appengine.api.datastore import Key, datastore_errors -from .models import GAEKey, GAEAncestorKey -class GAEKeyField(models.Field): - description = "A field for Google AppEngine Key objects" +import logging + +class DbKeyField(models.Field): + description = "A field for native database key objects" __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): + kwargs['null'] = True kwargs['blank'] = True + self.parent_key_attname = kwargs.pop('parent_key_name', None) if self.parent_key_attname is not None and kwargs.get('primary_key', None) is None: - raise ValueError("Primary key must be true to set parent_key_name") + raise ValueError("Primary key must be true to use parent_key_name") - super(GAEKeyField, self).__init__(*args, **kwargs) + super(DbKeyField, self).__init__(*args, **kwargs) def contribute_to_class(self, cls, name): if self.primary_key: @@ -27,57 +33,54 @@ def contribute_to_class(self, cls, name): def get_parent_key(instance, instance_type=None): if instance is None: return self + return instance.__dict__.get(self.parent_key_attname) def set_parent_key(instance, value): if instance is None: raise AttributeError("Attribute must be accessed via instance") - if not isinstance(value, GAEKey): - raise ValueError("parent must be a GAEKey") + if not isinstance(value, Key): + raise ValueError("'%s' must be a Key" % self.parent_key_attname) instance.__dict__[self.parent_key_attname] = value setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) - super(GAEKeyField, self).contribute_to_class(cls, name) + super(DbKeyField, self).contribute_to_class(cls, name) def to_python(self, value): if value is None: return None - if isinstance(value, basestring) and len(value) == 0: - return None - if isinstance(value, GAEKey): - return value if isinstance(value, Key): - return GAEKey(real_key=value) + return value if isinstance(value, basestring): try: - return GAEKey(real_key=Key(encoded=value)) + return Key(encoded=value) except datastore_errors.BadKeyError: - return GAEKey(real_key=Key.from_path(self.model._meta.db_table, long(value))) + return Key.from_path(self.model._meta.db_table, long(value)) if isinstance(value, (int, long)): - return GAEKey(real_key=Key.from_path(self.model._meta.db_table, value)) + return Key.from_path(self.model._meta.db_table, value) - raise ValidationError("GAEKeyField does not accept %s" % type(value)) + raise ValidationError("DbKeyField does not accept %s" % type(value)) - def get_prep_value(self, value): - if isinstance(value, GAEAncestorKey): - return value - return self.to_python(value) + def pre_save(self, model_instance, add): + value = super(DbKeyField, self).pre_save(model_instance, add) - def formfield(self, **kwargs): - return None + if add and value is None and self.parent_key_attname is not None and hasattr(model_instance, self.parent_key_attname): + stashed_parent = getattr(model_instance, self.parent_key_attname) + value = Key.from_path(self.model._meta.db_table, 0, parent=stashed_parent) - def pre_save(self, model_instance, add): - if add and self.parent_key_attname is not None: - parent_key = getattr(model_instance, self.parent_key_attname) - if parent_key is not None: - key = GAEKey(parent_key=parent_key) - setattr(model_instance, self.attname, key) - return key + return value - return super(GAEKeyField, self).pre_save(model_instance, add) + def get_prep_lookup(self, lookup_type, value): + if not isinstance(value, (Key, AncestorKey)): + raise ValueError(u"'%s' only accepts Key or ancestor objects, not %s" % (self.name, type(value))) + + return value + + def formfield(self, **kwargs): + return None def value_to_string(self, obj): - return smart_unicode(self._get_val_from_obj(obj).real_key()) + return smart_unicode(self._get_val_from_obj(obj)) diff --git a/models.py b/models.py index b1fe7e2..e69de29 100644 --- a/models.py +++ b/models.py @@ -1,75 +0,0 @@ -from django.db import models -from google.appengine.api.datastore import Key - -class GAEAncestorKey(object): - def __init__(self, key): - if not isinstance(key, Key): - raise ValueError('key must be of type Key') - - self._key = key - - def key(self): - return self._key - -class GAEKey(object): - def __init__(self, id_or_name=None, parent_key=None, real_key=None): - self._id_or_name = id_or_name - self._parent_key = parent_key - self._real_key = None - - if real_key is not None: - if id_or_name is not None or parent_key is not None: - raise ValueError("You can't set both a real_key and an id_or_name or parent_key") - - self._real_key = real_key - if real_key.parent(): - self._parent_key = GAEKey(real_key=real_key.parent()) - self._id_or_name = real_key.id_or_name() - - def id_or_name(self): - return self._id_or_name - - def parent_key(self): - return self._parent_key - - def real_key(self): - if self._real_key is None: - raise AttributeError("Incomplete key, please save the entity first.") - return self._real_key - - def has_real_key(self): - return self._real_key is not None - - def as_ancestor(self): - return GAEAncestorKey(self.real_key()) - - def __cmp__(self, other): - if not isinstance(other, GAEKey): - return 1 - - if self._real_key is not None and other._real_key is not None: - return cmp(self._real_key, other._real_key) - - if self._id_or_name is None or other._id_or_name is None: - raise ValueError("You can't compare unsaved keys: %s %s" % (self, other)) - - result = 0 - if self._parent_key is not None: - result = cmp(self._parent_key, other._parent_key) - - if result == 0: - result = cmp(self._id_or_name, other._id_or_name) - - return result - - def __hash__(self): - if self._real_key is None: - raise ValueError("You can't hash an unsaved key.") - - return hash(self._real_key) - - def __str__(self): - return str(self._id_or_name) - - def __repr__(self): - return "%s(id_or_name=%r, parent_key=%r, real_key=%r)" % (self.__class__, self._id_or_name, self._parent_key, self._real_key) diff --git a/tests/__init__.py b/tests/__init__.py index 9851772..7da57e2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,9 +1,9 @@ from .backend import BackendTest +from .decimals import DecimalTest from .field_db_conversion import FieldDBConversionTest from .field_options import FieldOptionsTest from .filter import FilterTest -from .keys import KeysTest +from .keys import KeysTest, DbKeyFieldTest, AncestorQueryTest, ParentKeyTest from .not_return_sets import NonReturnSetsTest from .order import OrderTest from .transactions import TransactionTest -from .keys import AncestorKeysTest, KeysTest diff --git a/tests/keys.py b/tests/keys.py index 517ce37..4603435 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -6,12 +6,12 @@ from django.test import TestCase from django.utils import unittest +from djangoappengine.fields import DbKeyField +from djangoappengine.db.utils import as_ancestor from djangotoolbox.fields import ListField from google.appengine.api.datastore import Key -from ..fields import GAEKeyField -from ..models import GAEKey class AutoKey(models.Model): @@ -295,41 +295,49 @@ def test_key_kind(self): class ParentModel(models.Model): - key = GAEKeyField(primary_key=True) + key = DbKeyField(primary_key=True) -class NonGAEParentModel(models.Model): +class NonDbKeyParentModel(models.Model): id = models.AutoField(primary_key=True) class ChildModel(models.Model): - key = GAEKeyField(primary_key=True, parent_key_name='parent_key') + key = DbKeyField(primary_key=True, parent_key_name='parent_key') class AnotherChildModel(models.Model): - key = GAEKeyField(primary_key=True, parent_key_name='also_parent_key') + key = DbKeyField(primary_key=True, parent_key_name='parent_key') class ForeignKeyModel(models.Model): id = models.AutoField(primary_key=True) relation = models.ForeignKey(ParentModel) -class AncestorKeysTest(TestCase): - def testGAEKeySave(self): +class DbKeyFieldTest(TestCase): + def testDbKeySave(self): model = ParentModel() model.save() self.assertIsNotNone(model.pk) - def testUnsavedParent(self): + def testForeignKeyWithGAEKey(self): parent = ParentModel() + parent.save() - with self.assertRaises(ValueError): - child = ChildModel(parent_key=parent.pk) + fkm = ForeignKeyModel() + fkm.relation = parent + fkm.save() - def testNonGAEParent(self): - parent = NonGAEParentModel() + results = list(ForeignKeyModel.objects.filter(relation=parent)) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, fkm.pk) + + def testPrimaryKeyQuery(self): + parent = ParentModel() parent.save() - with self.assertRaises(ValueError): - child = ChildModel(parent_key=parent.pk) + db_parent = ParentModel.objects.get(pk=parent.pk) + self.assertEquals(parent.pk, db_parent.pk) + +class ParentKeyTest(TestCase): def testParentChildSave(self): parent = ParentModel() orig_parent_pk = parent.pk @@ -341,9 +349,24 @@ def testParentChildSave(self): self.assertNotEquals(parent.pk, orig_parent_pk) self.assertNotEquals(child.pk, orig_child_pk) - self.assertEquals(child.pk.parent_key(), parent.pk) - self.assertEquals(child.pk.parent_key().real_key(), parent.pk.real_key()) + self.assertEquals(child.pk.parent(), parent.pk) + + def testParentModelChildSave(self): + parent = ParentModel() + orig_parent_pk = parent.pk + parent.save() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent) + + def testNonDbKeyParent(self): + parent = NonDbKeyParentModel() + parent.save() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent.pk) +class AncestorQueryTest(TestCase): def testAncestorFilterQuery(self): parent = ParentModel() parent.save() @@ -351,7 +374,7 @@ def testAncestorFilterQuery(self): child = ChildModel(parent_key=parent.pk) child.save() - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) self.assertEquals(1, len(results)) self.assertEquals(results[0].pk, child.pk) @@ -363,7 +386,7 @@ def testAncestorGetQuery(self): child = ChildModel(parent_key=parent.pk) child.save() - result = ChildModel.objects.get(pk=parent.pk.as_ancestor()) + result = ChildModel.objects.get(pk=as_ancestor(parent.pk)) self.assertEquals(result.pk, child.pk) @@ -371,7 +394,7 @@ def testEmptyAncestorQuery(self): parent = ParentModel() parent.save() - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) self.assertEquals(0, len(results)) @@ -381,15 +404,15 @@ def testEmptyAncestorQueryWithUnsavedChild(self): child = ChildModel(parent_key=parent.pk) - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) self.assertEquals(0, len(results)) def testUnsavedAncestorQuery(self): parent = ParentModel() - with self.assertRaises(AttributeError): - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + with self.assertRaises(ValueError): + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) def testDifferentChildrenAncestorQuery(self): parent = ParentModel() @@ -397,17 +420,17 @@ def testDifferentChildrenAncestorQuery(self): child1 = ChildModel(parent_key=parent.pk) child1.save() - child2 = AnotherChildModel(also_parent_key=parent.pk) + child2 = AnotherChildModel(parent_key=parent.pk) child2.save() - results = list(ChildModel.objects.filter(pk=parent.pk.as_ancestor())) + results1 = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, child1.pk) + self.assertEquals(1, len(results1)) + self.assertEquals(results1[0].pk, child1.pk) - results = list(AnotherChildModel.objects.filter(pk=parent.pk.as_ancestor())) - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, child2.pk) + results2 = list(AnotherChildModel.objects.filter(pk=as_ancestor(parent.pk))) + self.assertEquals(1, len(results2)) + self.assertEquals(results2[0].pk, child2.pk) def testDifferentParentsAncestorQuery(self): parent1 = ParentModel() @@ -422,47 +445,11 @@ def testDifferentParentsAncestorQuery(self): child2 = ChildModel(parent_key=parent2.pk) child2.save() - results = list(ChildModel.objects.filter(pk=parent1.pk.as_ancestor())) - - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, child1.pk) + results1 = list(ChildModel.objects.filter(pk=as_ancestor(parent1.pk))) - results = list(ChildModel.objects.filter(pk=parent2.pk.as_ancestor())) - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, child2.pk) + self.assertEquals(1, len(results1)) + self.assertEquals(results1[0].pk, child1.pk) - def testForeignKeyWithGAEKey(self): - parent = ParentModel() - parent.save() - - fkm = ForeignKeyModel() - fkm.relation = parent - fkm.save() - - results = list(ForeignKeyModel.objects.filter(relation=parent)) - self.assertEquals(1, len(results)) - self.assertEquals(results[0].pk, fkm.pk) - - def testPrimaryKeyQuery(self): - parent = ParentModel() - parent.save() - - db_parent = ParentModel.objects.get(pk=parent.pk) - - self.assertEquals(parent.pk, db_parent.pk) - - def testPrimaryKeyQueryStringKey(self): - parent = ParentModel() - parent.save() - - db_parent = ParentModel.objects.get(pk=str(parent.pk)) - - self.assertEquals(parent.pk, db_parent.pk) - - def testPrimaryKeyQueryIntKey(self): - parent = ParentModel() - parent.save() - - db_parent = ParentModel.objects.get(pk=int(str(parent.pk))) - - self.assertEquals(parent.pk, db_parent.pk) + results2 = list(ChildModel.objects.filter(pk=as_ancestor(parent2.pk))) + self.assertEquals(1, len(results2)) + self.assertEquals(results2[0].pk, child2.pk) From 7407854cebd3a90d00fa868e2f7c87ac89d952b9 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Thu, 19 Apr 2012 21:45:26 -0400 Subject: [PATCH 17/55] DBKeyField is a special case for conversion --- db/creation.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db/creation.py b/db/creation.py index 4848190..c1749b4 100644 --- a/db/creation.py +++ b/db/creation.py @@ -32,6 +32,13 @@ def db_type(self, field): field is to be indexed, and the "text" db_type (db.Text) if it's registered as unindexed. """ + from djangoappengine.fields import DbKeyField + + # DBKeyField reads/stores db.Key objects directly + # so its treated as a special case + if isinstance(field, DbKeyField): + return field.db_type(connection=self.connection) + if self.connection.settings_dict.get('STORE_RELATIONS_AS_DB_KEYS'): if field.primary_key or field.rel is not None: return 'key' From b77c18306973391eb9aba99935ae70494e0dc4cd Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 20 Apr 2012 14:10:15 -0400 Subject: [PATCH 18/55] add missing import --- db/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/compiler.py b/db/compiler.py index 0bb5e0b..0ce7d23 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -22,7 +22,7 @@ from .db_settings import get_model_indexes from .expressions import ExpressionEvaluator -from .utils import commit_locked +from .utils import AncestorKey, commit_locked # Valid query types (a dictionary is used for speedy lookups). From 9be9b5f634e075be8a72adc5036bc21c5ef37e99 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 20 Apr 2012 14:16:40 -0400 Subject: [PATCH 19/55] dbkeyfield should not be forced to nullable --- fields.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fields.py b/fields.py index e013482..54197d5 100644 --- a/fields.py +++ b/fields.py @@ -13,7 +13,6 @@ class DbKeyField(models.Field): __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): - kwargs['null'] = True kwargs['blank'] = True self.parent_key_attname = kwargs.pop('parent_key_name', None) @@ -55,6 +54,9 @@ def to_python(self, value): if isinstance(value, Key): return value if isinstance(value, basestring): + if len(value) == 0: + return None + try: return Key(encoded=value) except datastore_errors.BadKeyError: @@ -64,6 +66,11 @@ def to_python(self, value): raise ValidationError("DbKeyField does not accept %s" % type(value)) + def get_prep_value(self, value): + if isinstance(value, AncestorKey): + return value + return self.to_python(value) + def pre_save(self, model_instance, add): value = super(DbKeyField, self).pre_save(model_instance, add) @@ -73,12 +80,6 @@ def pre_save(self, model_instance, add): return value - def get_prep_lookup(self, lookup_type, value): - if not isinstance(value, (Key, AncestorKey)): - raise ValueError(u"'%s' only accepts Key or ancestor objects, not %s" % (self.name, type(value))) - - return value - def formfield(self, **kwargs): return None From 4c6ebdf6cb80fa8c670b69ef4d48901290d96ad3 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Fri, 20 Apr 2012 14:17:20 -0400 Subject: [PATCH 20/55] added make_key function to simplify created DbKeys from models --- db/utils.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/db/utils.py b/db/utils.py index e324697..f7540be 100644 --- a/db/utils.py +++ b/db/utils.py @@ -64,10 +64,30 @@ class AncestorKey(object): def __init__(self, key): self.key = key -def as_ancestor(key): - if key is None: - raise ValueError("key must not be None") - return AncestorKey(key) +def as_ancestor(key_or_model): + if key_or_model is None: + raise ValueError("key_or_model must not be None") -def make_key(model, id_or_name, parent=None): - return Key.from_path(model._meta.db_table, id_or_name, parent=parent) + if isinstance(key_or_model, models.Model): + key_or_model = Key.from_path(key_or_model._meta.db_table, key_or_model.pk) + + return AncestorKey(key_or_model) + +def make_key(*args, **kwargs): + parent = kwargs.pop('parent', None) + + if kwargs: + raise AssertionError('Excess keyword arguments; received %s' % kwargs) + + if not args or len(args) % 2: + raise AssertionError('A non-zero even number of positional arguments is required; received %s' % args) + + if isinstance(parent, models.Model): + parent = Key.from_path(parent._meta.db_table, parent.pk) + + converted_args = [] + for i in xrange(0, len(args), 2): + model, id_or_name = args[i:i+2] + converted_args.extend((model._meta.db_table, id_or_name)) + + return Key.from_path(*converted_args, parent=parent) From 13c8b64749a84043707b55b11cd948442eb2b6bf Mon Sep 17 00:00:00 2001 From: User Date: Wed, 23 May 2012 17:52:12 -0400 Subject: [PATCH 21/55] Merge branch 'feature/ancestor-query-1.4' of git://github.com/django-nonrel/djangoappengine into develop Conflicts: db/compiler.py --- db/creation.py | 9 +++++---- tests/__init__.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/db/creation.py b/db/creation.py index c1749b4..7a269dc 100644 --- a/db/creation.py +++ b/db/creation.py @@ -77,7 +77,8 @@ def _create_test_db(self, *args, **kw): stub_manager.activate_test_stubs() def _destroy_test_db(self, *args, **kw): - if self._had_test_stubs: - stub_manager.deactivate_test_stubs() - stub_manager.setup_stubs(self.connection) - del self._had_test_stubs + if hasattr(self, '_had_test_stubs'): + if self._had_test_stubs: + stub_manager.deactivate_test_stubs() + stub_manager.setup_stubs(self.connection) + del self._had_test_stubs diff --git a/tests/__init__.py b/tests/__init__.py index 7da57e2..fb3bf09 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,5 @@ from .backend import BackendTest -from .decimals import DecimalTest +#from .decimals import DecimalTest from .field_db_conversion import FieldDBConversionTest from .field_options import FieldOptionsTest from .filter import FilterTest From a417ac0cbc2284a1c6a59da3fc6663040af94354 Mon Sep 17 00:00:00 2001 From: User Date: Fri, 25 May 2012 17:45:38 -0400 Subject: [PATCH 22/55] Add some syntactic sugar classes for doing ancestor queries. --- db/models/__init__.py | 0 db/models/manager.py | 14 +++++++++++++ db/models/query.py | 17 ++++++++++++++++ test.py | 32 +++++++++++++++++++++++++++++ tests/__init__.py | 1 + tests/ancestor.py | 47 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 111 insertions(+) create mode 100644 db/models/__init__.py create mode 100644 db/models/manager.py create mode 100644 db/models/query.py create mode 100644 test.py create mode 100644 tests/ancestor.py diff --git a/db/models/__init__.py b/db/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db/models/manager.py b/db/models/manager.py new file mode 100644 index 0000000..5b5e732 --- /dev/null +++ b/db/models/manager.py @@ -0,0 +1,14 @@ +from django.db.models import Manager as _baseManager +from djangoappengine.db.utils import as_ancestor +from djangoappengine.db.models.query import QuerySet + +class Manager(_baseManager): + + def get_query_set(self): + """Returns a new QuerySet object. Subclasses can override this method + to easily customize the behavior of the Manager. + """ + return QuerySet(self.model, using=self._db) + + def ancestor(self, ancestor): + return self.get_query_set().ancestor(ancestor) \ No newline at end of file diff --git a/db/models/query.py b/db/models/query.py new file mode 100644 index 0000000..d81c661 --- /dev/null +++ b/db/models/query.py @@ -0,0 +1,17 @@ +from django.db.models.query import QuerySet as _baseQuerySet +from djangoappengine.db.utils import as_ancestor + +class QuerySet(_baseQuerySet): + def ancestor(self, ancestor): + """ + Returns a new QuerySet instance with the args ANDed to the existing + set. + """ + return self._filter_or_exclude(False, pk=as_ancestor(ancestor)) + +class EmptyQuerySet(QuerySet): + def ancestor(self, *args, **kwargs): + """ + Always returns EmptyQuerySet. + """ + return self \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..ae8a773 --- /dev/null +++ b/test.py @@ -0,0 +1,32 @@ +from django.test import TestCase + +from google.appengine.datastore import datastore_stub_util + +from db.stubs import stub_manager + +class GAETestCase(TestCase): + def _pre_setup(self): + """Performs any pre-test setup. + * Set the dev_appserver consistency state. + """ + super(GAETestCase,self)._pre_setup() + + if hasattr(self, 'consistency_probability'): + datastore = stub_manager.testbed.get_stub('datastore_v3') + self._orig_policy = datastore._consistency_policy + + datastore.SetConsistencyPolicy(datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=self.consistency_probability)) + + + def _post_teardown(self): + """ Performs any post-test things. This includes: + + * Putting back the original ROOT_URLCONF if it was changed. + * Force closing the connection, so that the next test gets + a clean cursor. + """ + if hasattr(self, '_orig_policy'): + datastore = stub_manager.testbed.get_stub('datastore_v3') + datastore.SetConsistencyPolicy(self._orig_policy) + + super(GAETestCase,self)._post_teardown() diff --git a/tests/__init__.py b/tests/__init__.py index fb3bf09..75b8a05 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,3 +7,4 @@ from .not_return_sets import NonReturnSetsTest from .order import OrderTest from .transactions import TransactionTest +from .ancestor import AncestorTest diff --git a/tests/ancestor.py b/tests/ancestor.py new file mode 100644 index 0000000..0f4b815 --- /dev/null +++ b/tests/ancestor.py @@ -0,0 +1,47 @@ +from django.test import TestCase +from django.utils import unittest +from django.db import models + +from djangoappengine.fields import DbKeyField + +from djangoappengine.db.models.manager import Manager + +#from djangotoolbox.fields import ListField +#from google.appengine.api.datastore import Key + +class ParentFoo(models.Model): + key = DbKeyField(primary_key=True) + foo = models.IntegerField() + objects = Manager() + +class ChildFoo(models.Model): + key = DbKeyField(primary_key=True, parent_key_name='parent_key') + foo = models.IntegerField() + objects = Manager() + +class AncestorTest(TestCase): + def test_simple(self): + px = ParentFoo(foo=5) + px.save() + px = ParentFoo(foo=2) + px.save() + + parents = ParentFoo.objects.all() + self.assertEqual(2, parents.count()) + + parents = ParentFoo.objects.filter(foo=2) + self.assertEqual(1, parents.count()) + + child = ChildFoo(foo=10, parent_key=px.pk) + orig_child_pk = child.pk + child.save() + + results = list(ChildFoo.objects.ancestor(px.pk)) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) + + results = list(ChildFoo.objects.all().ancestor(px.pk)) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) \ No newline at end of file From e900212662b2b7c6ee672fad5a0a91c0986bbdca Mon Sep 17 00:00:00 2001 From: User Date: Sat, 26 May 2012 15:20:18 -0400 Subject: [PATCH 23/55] Put datastore persistence patch back in, it's still necessary for 1.6.5 --- boot.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/boot.py b/boot.py index 4abdc5e..d96e156 100644 --- a/boot.py +++ b/boot.py @@ -174,6 +174,13 @@ def setup_project(): logging.warn("Could not patch modules whitelist. the compiler " "and parser modules will not work and SSL support " "is disabled.") + # In SDK 1.6.4, the datastore doesn't save automatically on exit. + # Register a handler to make sure we save. This is important on + # manage.py commands other than 'runserver'. Note that with runserver, + # the datastore is flushed twice. This should be acceptable. + import atexit + if hasattr(dev_appserver, 'TearDownStubs'): + atexit.register(dev_appserver.TearDownStubs) elif not on_production_server: try: # Restore the real subprocess module. From dd03270a21443436bbf6d0f975cfb7c32f6c3aff Mon Sep 17 00:00:00 2001 From: User Date: Wed, 30 May 2012 02:08:38 -0400 Subject: [PATCH 24/55] Also fix db/utils.py so it works in python 2.5 It's possible to use a django DateField or TimeField but have your own encoding in the datastore. In this case, we may not get a datetime value, so check type first. --- db/base.py | 4 ++-- db/utils.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/db/base.py b/db/base.py index 631bf36..bf563d7 100644 --- a/db/base.py +++ b/db/base.py @@ -208,9 +208,9 @@ def _value_from_db(self, value, field, field_kind, db_type): value = unicode(value) # Dates and times are stored as datetimes, drop the added part. - elif db_type == 'date': + elif db_type == 'date' and isinstance(value, datetime.datetime): value = value.date() - elif db_type == 'time': + elif db_type == 'time' and isinstance(value, datetime.datetime): value = value.time() # Convert GAE Blobs to plain strings for Django. diff --git a/db/utils.py b/db/utils.py index f7540be..576c635 100644 --- a/db/utils.py +++ b/db/utils.py @@ -90,4 +90,5 @@ def make_key(*args, **kwargs): model, id_or_name = args[i:i+2] converted_args.extend((model._meta.db_table, id_or_name)) - return Key.from_path(*converted_args, parent=parent) + newkwargs = { 'parent' : parent } + return Key.from_path(*converted_args, **newkwargs) From b38cb39733b6b142c130a081f85503b8c1160d01 Mon Sep 17 00:00:00 2001 From: User Date: Fri, 29 Jun 2012 16:44:13 -0400 Subject: [PATCH 25/55] db_type may be marked "date"/"time", as long as it's derived from DateField or TimeField, but may not necessarily be stored as a datetime. --- db/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/base.py b/db/base.py index bf563d7..884b575 100644 --- a/db/base.py +++ b/db/base.py @@ -170,9 +170,9 @@ def _value_for_db(self, value, field, field_kind, db_type, lookup): # Store all date / time values as datetimes, by using some # default time or date. - elif db_type == 'date': + elif db_type == 'date' and isinstance(value, datetime.date): value = datetime.datetime.combine(value, self.DEFAULT_TIME) - elif db_type == 'time': + elif db_type == 'time' and isinstance(value, datetime.time): value = datetime.datetime.combine(self.DEFAULT_DATE, value) # Store BlobField, DictField and EmbeddedModelField values as Blobs. From f592f4e9be5a20e76c866e9cd8f400a027467b53 Mon Sep 17 00:00:00 2001 From: User Date: Fri, 13 Jul 2012 01:21:43 -0400 Subject: [PATCH 26/55] Add dev_appserver compatible version of LiveServerTestCase. Patch the stub initialization to use the default dev_appserver login url for the user service. Otherwise it defaults to the live production user service, which we generally don't want while testing. --- db/stubs.py | 2 +- test.py | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/db/stubs.py b/db/stubs.py index f3c3258..36c78d2 100644 --- a/db/stubs.py +++ b/db/stubs.py @@ -48,7 +48,7 @@ def activate_test_stubs(self): self.testbed.init_memcache_stub() self.testbed.init_taskqueue_stub(root_path=PROJECT_DIR) self.testbed.init_urlfetch_stub() - self.testbed.init_user_stub() + self.testbed.init_user_stub(True, **{ 'login_url' : '/_ah/login?continue=%s' }) self.testbed.init_xmpp_stub() self.testbed.init_channel_stub() diff --git a/test.py b/test.py index ae8a773..1427735 100644 --- a/test.py +++ b/test.py @@ -1,10 +1,23 @@ -from django.test import TestCase +import os +import threading +import httplib +from django.core.handlers.wsgi import WSGIHandler +from django.core.servers.basehttp import WSGIServerException +from django.db import connections +from django.test import TestCase, TransactionTestCase +from django.test.testcases import _MediaFilesHandler +from django.contrib.staticfiles.handlers import StaticFilesHandler +from google.appengine.tools import dev_appserver +from google.appengine.tools import dev_appserver_main from google.appengine.datastore import datastore_stub_util from db.stubs import stub_manager class GAETestCase(TestCase): + ''' + This base class configures the dev_appserver datastore to test for eventual consistency behavior. + ''' def _pre_setup(self): """Performs any pre-test setup. * Set the dev_appserver consistency state. @@ -30,3 +43,160 @@ def _post_teardown(self): datastore.SetConsistencyPolicy(self._orig_policy) super(GAETestCase,self)._post_teardown() + + +class LiveServerThread(threading.Thread): + """ + Thread for running a live http server while the tests are running. + + This is mostly copied from django.test.testcases.LiveServerThread + It's modified slightly to launch dev_appserver instead of a plain + HTTP server. The shutdown mechanism is slightly different too. + """ + + def __init__(self, host, possible_ports, connections_override=None): + self.host = host + self.port = None + self.possible_ports = possible_ports + self.is_ready = threading.Event() + self.error = None + self.connections_override = connections_override + super(LiveServerThread, self).__init__() + + def run(self): + """ + Sets up the live server and databases, and then loops over handling + http requests. + """ + if self.connections_override: + from django.db import connections + # Override this thread's database connections with the ones + # provided by the main thread. + for alias, conn in self.connections_override.items(): + connections[alias] = conn + try: + # Create the handler for serving static and media files + handler = StaticFilesHandler(_MediaFilesHandler(WSGIHandler())) + + # Go through the list of possible ports, hoping that we can find + # one that is free to use for the WSGI server. + for index, port in enumerate(self.possible_ports): + try: + options = dev_appserver_main.DEFAULT_ARGS.copy() + dev_appserver.SetupStubs("project-eat", **options) + self.httpd = dev_appserver.CreateServer(".", '/_ah/login', port, default_partition="dev") + + except WSGIServerException, e: + if sys.version_info < (2, 6): + error_code = e.args[0].args[0] + else: + error_code = e.args[0].errno + if (index + 1 < len(self.possible_ports) and + error_code == errno.EADDRINUSE): + # This port is already in use, so we go on and try with + # the next one in the list. + continue + else: + # Either none of the given ports are free or the error + # is something else than "Address already in use". So + # we let that error bubble up to the main thread. + raise + else: + # A free port was found. + self.port = port + break + + #self.httpd.set_app(handler) + self.is_ready.set() + self.httpd.serve_forever() + except Exception, e: + self.error = e + self.is_ready.set() + + def join(self, timeout=None): + if hasattr(self, 'httpd'): + # Stop the WSGI server + self.httpd.stop_serving_forever() + try: + # We need to hit the server with one more request to make it quit + connection = httplib.HTTPConnection(self.host, self.port) + connection.request('GET',"/") + except: + pass + super(LiveServerThread, self).join(timeout) + +# This is copied directly from django.test.testcases +class LiveServerTestCase(TransactionTestCase): + """ + Does basically the same as TransactionTestCase but also launches a live + http server in a separate thread so that the tests may use another testing + framework, such as Selenium for example, instead of the built-in dummy + client. + Note that it inherits from TransactionTestCase instead of TestCase because + the threads do not share the same transactions (unless if using in-memory + sqlite) and each thread needs to commit all their transactions so that the + other thread can see the changes. + """ + + @property + def live_server_url(self): + return 'http://%s:%s' % ( + self.server_thread.host, self.server_thread.port) + + @classmethod + def setUpClass(cls): + connections_override = {} + for conn in connections.all(): + # If using in-memory sqlite databases, pass the connections to + # the server thread. + if (conn.settings_dict['ENGINE'] == 'django.db.backends.sqlite3' + and conn.settings_dict['NAME'] == ':memory:'): + # Explicitly enable thread-shareability for this connection + conn.allow_thread_sharing = True + connections_override[conn.alias] = conn + + # Launch the live server's thread + specified_address = os.environ.get( + 'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081') + + # The specified ports may be of the form '8000-8010,8080,9200-9300' + # i.e. a comma-separated list of ports or ranges of ports, so we break + # it down into a detailed list of all possible ports. + possible_ports = [] + try: + host, port_ranges = specified_address.split(':') + for port_range in port_ranges.split(','): + # A port range can be of either form: '8000' or '8000-8010'. + extremes = map(int, port_range.split('-')) + assert len(extremes) in [1, 2] + if len(extremes) == 1: + # Port range of the form '8000' + possible_ports.append(extremes[0]) + else: + # Port range of the form '8000-8010' + for port in range(extremes[0], extremes[1] + 1): + possible_ports.append(port) + except Exception: + raise ImproperlyConfigured('Invalid address ("%s") for live ' + 'server.' % specified_address) + cls.server_thread = LiveServerThread( + host, possible_ports, connections_override) + cls.server_thread.daemon = True + cls.server_thread.start() + + # Wait for the live server to be ready + cls.server_thread.is_ready.wait() + if cls.server_thread.error: + raise cls.server_thread.error + + super(LiveServerTestCase, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + # There may not be a 'server_thread' attribute if setUpClass() for some + # reasons has raised an exception. + if hasattr(cls, 'server_thread'): + # Terminate the live server's thread + cls.server_thread.join() + super(LiveServerTestCase, cls).tearDownClass() + From 8abaf2a0030e6bb9dc1b9fa5ebfe857237cc9a70 Mon Sep 17 00:00:00 2001 From: User Date: Sat, 14 Jul 2012 14:12:59 -0400 Subject: [PATCH 27/55] Hack around two threading issues with the LiveServerTestCase 1. The task queue stub spawns a new thread that causes badness. Disable task queue while testing to avoid the new thread. 2. If the launched browser is accessing the site and hitting the db, it may conflict with the main thread when it flushes the DB between tests. The real fix is to ensure the test doesn't complete on the main thread until the browser is done (this includes JS running in browser which may hit the server). I've put in a hackaround to keep retrying the db flush if it fails. This is not safe though. --- test.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/test.py b/test.py index 1427735..c12b443 100644 --- a/test.py +++ b/test.py @@ -1,6 +1,7 @@ import os import threading -import httplib +import httplib +import logging from django.core.handlers.wsgi import WSGIHandler from django.core.servers.basehttp import WSGIServerException from django.db import connections @@ -83,6 +84,7 @@ def run(self): for index, port in enumerate(self.possible_ports): try: options = dev_appserver_main.DEFAULT_ARGS.copy() + options['disable_task_running'] = True # Prevent launch of task queue thread dev_appserver.SetupStubs("project-eat", **options) self.httpd = dev_appserver.CreateServer(".", '/_ah/login', port, default_partition="dev") @@ -121,8 +123,10 @@ def join(self, timeout=None): # We need to hit the server with one more request to make it quit connection = httplib.HTTPConnection(self.host, self.port) connection.request('GET',"/") - except: - pass + connection.close() + self.httpd.server_close() + except Exception, e: + logging.error("LiveServerThread join caught " + str(e)) super(LiveServerThread, self).join(timeout) # This is copied directly from django.test.testcases @@ -200,3 +204,30 @@ def tearDownClass(cls): cls.server_thread.join() super(LiveServerTestCase, cls).tearDownClass() + def _pre_setup(self): + """Performs any pre-test setup. This includes: + + * Flushing the database. + * If the Test Case class has a 'fixtures' member, installing the + named fixtures. + * If the Test Case class has a 'urls' member, replace the + ROOT_URLCONF with it. + * Clearing the mail test outbox. + """ + retry = True + while retry: + ''' + There's some ugly multithreading bug where the db flush on the main thread (which runs tests) + fails because the child thread (running the http server) is doing something - I don't know what. + My hackaround is just to keep retrying the flush until it succeeds. + + There's already a warning in the djano docs about accessing the DB while the child thread is accessing it. + ''' + try: + super(LiveServerTestCase,self)._pre_setup() + retry = False + except Exception, e: + import eat + eat.stacktrace() + eat.gaebp(True) + From 637d8e1583914b004a77a91af9e7c26995ba0610 Mon Sep 17 00:00:00 2001 From: User Date: Mon, 16 Jul 2012 18:41:06 -0400 Subject: [PATCH 28/55] Make logout url work for LiveServerTest --- db/stubs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/stubs.py b/db/stubs.py index 36c78d2..0f214b9 100644 --- a/db/stubs.py +++ b/db/stubs.py @@ -48,7 +48,8 @@ def activate_test_stubs(self): self.testbed.init_memcache_stub() self.testbed.init_taskqueue_stub(root_path=PROJECT_DIR) self.testbed.init_urlfetch_stub() - self.testbed.init_user_stub(True, **{ 'login_url' : '/_ah/login?continue=%s' }) + self.testbed.init_user_stub(True, **{ 'logout_url' : '/_ah/login?continue=%s&action=Logout', + 'login_url' : '/_ah/login?continue=%s' }) self.testbed.init_xmpp_stub() self.testbed.init_channel_stub() From 05afb8c4c9505d9ff156f64514b2c55791742c01 Mon Sep 17 00:00:00 2001 From: User Date: Wed, 29 Aug 2012 22:17:10 -0400 Subject: [PATCH 29/55] Switch to WSGI for python 2.7 --- main/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/main/main.py b/main/main.py index 9ee5251..9d92334 100644 --- a/main/main.py +++ b/main/main.py @@ -71,7 +71,14 @@ def make_profileable(func): return lambda: profile_main(func) return func + +import webapp2 + +app = webapp2.WSGIApplication([('/', application)]) + +''' main = make_profileable(real_main) if __name__ == '__main__': main() +''' From 9f271c2d96ee73c5fc1a7d299e698076aca72a65 Mon Sep 17 00:00:00 2001 From: User Date: Thu, 4 Oct 2012 02:04:53 -0400 Subject: [PATCH 30/55] Add synchronization between the LiveServerTestCase and LiveServerThread. --- test.py | 81 +++++++++++++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/test.py b/test.py index c12b443..f3d63e2 100644 --- a/test.py +++ b/test.py @@ -1,6 +1,8 @@ import os +import types import threading import httplib +import select import logging from django.core.handlers.wsgi import WSGIHandler from django.core.servers.basehttp import WSGIServerException @@ -30,7 +32,6 @@ def _pre_setup(self): self._orig_policy = datastore._consistency_policy datastore.SetConsistencyPolicy(datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=self.consistency_probability)) - def _post_teardown(self): """ Performs any post-test things. This includes: @@ -45,6 +46,7 @@ def _post_teardown(self): super(GAETestCase,self)._post_teardown() +liveServerLock = threading.Lock() class LiveServerThread(threading.Thread): """ @@ -69,6 +71,21 @@ def run(self): Sets up the live server and databases, and then loops over handling http requests. """ + + def sync_handle_request(self): + try: + readable, _, _ = select.select([self.socket], [], [], 10) + if readable: + liveServerLock.acquire() + try: + self.original_handle_request() + except Exception, e: + pass + finally: + liveServerLock.release() + except Exception, e: + pass + if self.connections_override: from django.db import connections # Override this thread's database connections with the ones @@ -86,6 +103,7 @@ def run(self): options = dev_appserver_main.DEFAULT_ARGS.copy() options['disable_task_running'] = True # Prevent launch of task queue thread dev_appserver.SetupStubs("project-eat", **options) + self.httpd = dev_appserver.CreateServer(".", '/_ah/login', port, default_partition="dev") except WSGIServerException, e: @@ -109,13 +127,20 @@ def run(self): break #self.httpd.set_app(handler) + + # hack replace the http server with our sync enabled version + self.httpd.original_handle_request = self.httpd.handle_request + self.httpd.handle_request = types.MethodType(sync_handle_request, self.httpd) + self.is_ready.set() self.httpd.serve_forever() except Exception, e: + #logging.error("LiveServerThread caught exception " + str(e)) self.error = e self.is_ready.set() def join(self, timeout=None): + logging.info("LiveServerThread join called") if hasattr(self, 'httpd'): # Stop the WSGI server self.httpd.stop_serving_forever() @@ -141,14 +166,14 @@ class LiveServerTestCase(TransactionTestCase): sqlite) and each thread needs to commit all their transactions so that the other thread can see the changes. """ + lock = liveServerLock @property def live_server_url(self): return 'http://%s:%s' % ( self.server_thread.host, self.server_thread.port) - @classmethod - def setUpClass(cls): + def _pre_setup(self): connections_override = {} for conn in connections.all(): # If using in-memory sqlite databases, pass the connections to @@ -183,51 +208,23 @@ def setUpClass(cls): except Exception: raise ImproperlyConfigured('Invalid address ("%s") for live ' 'server.' % specified_address) - cls.server_thread = LiveServerThread( + self.server_thread = LiveServerThread( host, possible_ports, connections_override) - cls.server_thread.daemon = True - cls.server_thread.start() + self.server_thread.daemon = True + self.server_thread.start() # Wait for the live server to be ready - cls.server_thread.is_ready.wait() - if cls.server_thread.error: - raise cls.server_thread.error + self.server_thread.is_ready.wait() + if self.server_thread.error: + raise self.server_thread.error - super(LiveServerTestCase, cls).setUpClass() + super(LiveServerTestCase, self)._pre_setup() - @classmethod - def tearDownClass(cls): + def _post_teardown(self): # There may not be a 'server_thread' attribute if setUpClass() for some # reasons has raised an exception. - if hasattr(cls, 'server_thread'): + if hasattr(self, 'server_thread'): # Terminate the live server's thread - cls.server_thread.join() - super(LiveServerTestCase, cls).tearDownClass() - - def _pre_setup(self): - """Performs any pre-test setup. This includes: - - * Flushing the database. - * If the Test Case class has a 'fixtures' member, installing the - named fixtures. - * If the Test Case class has a 'urls' member, replace the - ROOT_URLCONF with it. - * Clearing the mail test outbox. - """ - retry = True - while retry: - ''' - There's some ugly multithreading bug where the db flush on the main thread (which runs tests) - fails because the child thread (running the http server) is doing something - I don't know what. - My hackaround is just to keep retrying the flush until it succeeds. - - There's already a warning in the djano docs about accessing the DB while the child thread is accessing it. - ''' - try: - super(LiveServerTestCase,self)._pre_setup() - retry = False - except Exception, e: - import eat - eat.stacktrace() - eat.gaebp(True) + self.server_thread.join() + super(LiveServerTestCase, self)._post_teardown() From e32ed1c7a5db61bef8bd775def8262c8b191aa3b Mon Sep 17 00:00:00 2001 From: User Date: Thu, 4 Oct 2012 16:40:16 -0400 Subject: [PATCH 31/55] Clean up comments in test utilities. --- test.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test.py b/test.py index f3d63e2..e0f7f67 100644 --- a/test.py +++ b/test.py @@ -16,6 +16,7 @@ from google.appengine.datastore import datastore_stub_util from db.stubs import stub_manager +from utils import appid class GAETestCase(TestCase): ''' @@ -55,6 +56,16 @@ class LiveServerThread(threading.Thread): This is mostly copied from django.test.testcases.LiveServerThread It's modified slightly to launch dev_appserver instead of a plain HTTP server. The shutdown mechanism is slightly different too. + + One big problem is that dev_appserver mangles the environment. + It's easy to run into threading issues where the dev_appserver + thread and the main application (test) thread conflict. One common + example is trying to use logging.error(), which will often cause + conflicts since dev_appserver replaces stderr. We use + liveServerLock to avoid these conflicts. + + Your own test code will need to acquire liveServerLock pretty much + every time you're doing something outside of an HTTP request. """ def __init__(self, host, possible_ports, connections_override=None): @@ -93,16 +104,13 @@ def sync_handle_request(self): for alias, conn in self.connections_override.items(): connections[alias] = conn try: - # Create the handler for serving static and media files - handler = StaticFilesHandler(_MediaFilesHandler(WSGIHandler())) - # Go through the list of possible ports, hoping that we can find # one that is free to use for the WSGI server. for index, port in enumerate(self.possible_ports): try: options = dev_appserver_main.DEFAULT_ARGS.copy() options['disable_task_running'] = True # Prevent launch of task queue thread - dev_appserver.SetupStubs("project-eat", **options) + dev_appserver.SetupStubs(appid, **options) self.httpd = dev_appserver.CreateServer(".", '/_ah/login', port, default_partition="dev") @@ -126,16 +134,14 @@ def sync_handle_request(self): self.port = port break - #self.httpd.set_app(handler) - - # hack replace the http server with our sync enabled version + # HACK: The magic happens here. We replace the http request handler + # with our sync'd version self.httpd.original_handle_request = self.httpd.handle_request self.httpd.handle_request = types.MethodType(sync_handle_request, self.httpd) self.is_ready.set() self.httpd.serve_forever() except Exception, e: - #logging.error("LiveServerThread caught exception " + str(e)) self.error = e self.is_ready.set() @@ -161,10 +167,14 @@ class LiveServerTestCase(TransactionTestCase): http server in a separate thread so that the tests may use another testing framework, such as Selenium for example, instead of the built-in dummy client. + Note that it inherits from TransactionTestCase instead of TestCase because the threads do not share the same transactions (unless if using in-memory sqlite) and each thread needs to commit all their transactions so that the other thread can see the changes. + + Be careful that almost everything you do needs to be synchronized against + the liveServerLock (which you can easily reference as this.lock() """ lock = liveServerLock From 69cfd34dc4365a17bae0459791f00d9cb889afbb Mon Sep 17 00:00:00 2001 From: User Date: Thu, 4 Oct 2012 17:05:28 -0400 Subject: [PATCH 32/55] Add input reader utility classes for use with the mapreduce API. The main benefit of both is to be able to pass in a Django model instead of a db.Model to get Keys or entities. Note that DjangoEntityInputReader returns a raw entity instead of a Django object instance. This is particularly useful for working with old entities in the datastore that may not match the latest Django model. Django's model classes could raise exceptions if properties were missing, but this class will allow fetching those entities. The developer can then add the missing properties, or whatever. --- mapreduce/inputreader.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 mapreduce/inputreader.py diff --git a/mapreduce/inputreader.py b/mapreduce/inputreader.py new file mode 100644 index 0000000..aeae8d1 --- /dev/null +++ b/mapreduce/inputreader.py @@ -0,0 +1,31 @@ +import djangoappengine.main + +from django.db.models.sql.query import Query + +from mapreduce.input_readers import AbstractDatastoreInputReader +from mapreduce import util +from google.appengine.datastore import datastore_query + +class DjangoKeyInputReader(AbstractDatastoreInputReader): + """An input reader that takes a Django model ('app.models.Model') and yields Keys for that model""" + + def _iter_key_range(self, k_range): + query = Query(util.for_name(self._entity_kind)).get_compiler(using="default").build_query() + raw_entity_kind = query.db_table + + query = k_range.make_ascending_datastore_query(raw_entity_kind, keys_only=True) + for key in query.Run( + config=datastore_query.QueryOptions(batch_size=self._batch_size)): + yield key, key + +class DjangoEntityInputReader(AbstractDatastoreInputReader): + """An input reader that takes a Django model ('app.models.Model') and yields entities for that model""" + + def _iter_key_range(self, k_range): + query = Query(util.for_name(self._entity_kind)).get_compiler(using="default").build_query() + raw_entity_kind = query.db_table + + query = k_range.make_ascending_datastore_query(raw_entity_kind) + for entity in query.Run( + config=datastore_query.QueryOptions(batch_size=self._batch_size)): + yield entity.key(), entity From eb29ef3894129484fc050877e33d5139512c9c70 Mon Sep 17 00:00:00 2001 From: User Date: Thu, 11 Oct 2012 21:31:52 -0400 Subject: [PATCH 33/55] Fix sync error between threads when loading fixtures. Fix socket retry code to find unused socket for next test. --- test.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/test.py b/test.py index e0f7f67..f181320 100644 --- a/test.py +++ b/test.py @@ -1,9 +1,10 @@ import os +import errno import types import threading import httplib +import socket import select -import logging from django.core.handlers.wsgi import WSGIHandler from django.core.servers.basehttp import WSGIServerException from django.db import connections @@ -17,6 +18,7 @@ from db.stubs import stub_manager from utils import appid +from unittest.runner import TextTestResult class GAETestCase(TestCase): ''' @@ -49,6 +51,12 @@ def _post_teardown(self): liveServerLock = threading.Lock() +class SyncTextTestResult(TextTestResult): + def addError(self, test, err): + if hasattr(test, "server_thread"): + test.server_thread.join() + super(SyncTextTestResult, self).addError(test, err) + class LiveServerThread(threading.Thread): """ Thread for running a live http server while the tests are running. @@ -113,12 +121,8 @@ def sync_handle_request(self): dev_appserver.SetupStubs(appid, **options) self.httpd = dev_appserver.CreateServer(".", '/_ah/login', port, default_partition="dev") - - except WSGIServerException, e: - if sys.version_info < (2, 6): - error_code = e.args[0].args[0] - else: - error_code = e.args[0].errno + except socket.error, e: + error_code = e.errno if (index + 1 < len(self.possible_ports) and error_code == errno.EADDRINUSE): # This port is already in use, so we go on and try with @@ -129,6 +133,7 @@ def sync_handle_request(self): # is something else than "Address already in use". So # we let that error bubble up to the main thread. raise + else: # A free port was found. self.port = port @@ -143,21 +148,24 @@ def sync_handle_request(self): self.httpd.serve_forever() except Exception, e: self.error = e + try: + self.httpd.server_close() + except Exception, e: + pass self.is_ready.set() def join(self, timeout=None): - logging.info("LiveServerThread join called") if hasattr(self, 'httpd'): # Stop the WSGI server - self.httpd.stop_serving_forever() try: + self.httpd.stop_serving_forever() # We need to hit the server with one more request to make it quit connection = httplib.HTTPConnection(self.host, self.port) connection.request('GET',"/") connection.close() - self.httpd.server_close() + #self.httpd.server_close() except Exception, e: - logging.error("LiveServerThread join caught " + str(e)) + pass super(LiveServerThread, self).join(timeout) # This is copied directly from django.test.testcases @@ -196,10 +204,10 @@ def _pre_setup(self): # Launch the live server's thread specified_address = os.environ.get( - 'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081') + 'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081-8089') # The specified ports may be of the form '8000-8010,8080,9200-9300' - # i.e. a comma-separated list of ports or ranges of ports, so we break + # i.e. a comma-separated list of ports or ranges gg ports, so we break # it down into a detailed list of all possible ports. possible_ports = [] try: @@ -218,6 +226,7 @@ def _pre_setup(self): except Exception: raise ImproperlyConfigured('Invalid address ("%s") for live ' 'server.' % specified_address) + self.server_thread = LiveServerThread( host, possible_ports, connections_override) self.server_thread.daemon = True @@ -228,7 +237,9 @@ def _pre_setup(self): if self.server_thread.error: raise self.server_thread.error + liveServerLock.acquire() # Lock while we load fixtures super(LiveServerTestCase, self)._pre_setup() + liveServerLock.release() def _post_teardown(self): # There may not be a 'server_thread' attribute if setUpClass() for some From 3139a05b98b128955308eab02b2094861d6a19df Mon Sep 17 00:00:00 2001 From: projecteat Date: Fri, 8 Feb 2013 02:17:59 -0500 Subject: [PATCH 34/55] A bunch of hacks to get django-nonrel serving on devappserver2. This is really limited, a lot of things don't work (like tests and dumpdata) --- boot.py | 77 ++++++++++++++++++-------------- db/stubs.py | 31 ++++++++----- management/commands/runserver.py | 39 ++++++++++++---- utils.py | 11 ++++- 4 files changed, 105 insertions(+), 53 deletions(-) diff --git a/boot.py b/boot.py index eda6176..555ca9b 100644 --- a/boot.py +++ b/boot.py @@ -69,8 +69,12 @@ def setup_env(): sys.path = [ sdk_path ] + sys.path # Then call fix_sys_path from the SDK - from dev_appserver import fix_sys_path - fix_sys_path() + try: + from dev_appserver import fix_sys_path + fix_sys_path() + except ImportError: + import devappserver2 + devappserver2.fix_sys_path(devappserver2.DEVAPPSERVER2_PATHS) setup_project() from .utils import have_appserver @@ -146,45 +150,51 @@ def setup_project(): # enable https connections (seem to be broken on Windows because # the _ssl module is disallowed). if not have_appserver: - from google.appengine.tools import dev_appserver - try: - # Backup os.environ. It gets overwritten by the - # dev_appserver, but it's needed by the subprocess module. - env = dev_appserver.DEFAULT_ENV - dev_appserver.DEFAULT_ENV = os.environ.copy() - dev_appserver.DEFAULT_ENV.update(env) - # Backup the buffer() builtin. The subprocess in Python 2.5 - # on Linux and OS X uses needs it, but the dev_appserver - # removes it. - dev_appserver.buffer = buffer - except AttributeError: - logging.warn("Could not patch the default environment. " - "The subprocess module will not work correctly.") - try: - # Allow importing compiler/parser, _ssl (for https), - # _io for Python 2.7 io support on OS X - dev_appserver.HardenedModulesHook._WHITE_LIST_C_MODULES.extend( - ('parser', '_ssl', '_io')) - except AttributeError: - logging.warn("Could not patch modules whitelist. the compiler " - "and parser modules will not work and SSL support " - "is disabled.") - # In SDK 1.6.4, the datastore doesn't save automatically on exit. - # Register a handler to make sure we save. This is important on - # manage.py commands other than 'runserver'. Note that with runserver, - # the datastore is flushed twice. This should be acceptable. - import atexit - if hasattr(dev_appserver, 'TearDownStubs'): - atexit.register(dev_appserver.TearDownStubs) + from google.appengine.tools import dev_appserver + try: + # Backup os.environ. It gets overwritten by the + # dev_appserver, but it's needed by the subprocess module. + env = dev_appserver.DEFAULT_ENV + dev_appserver.DEFAULT_ENV = os.environ.copy() + dev_appserver.DEFAULT_ENV.update(env) + # Backup the buffer() builtin. The subprocess in Python 2.5 + # on Linux and OS X uses needs it, but the dev_appserver + # removes it. + dev_appserver.buffer = buffer + except AttributeError: + logging.warn("Could not patch the default environment. " + "The subprocess module will not work correctly.") + + try: + # Allow importing compiler/parser, _ssl (for https), + # _io for Python 2.7 io support on OS X + dev_appserver.HardenedModulesHook._WHITE_LIST_C_MODULES.extend( + ('parser', '_ssl', '_io')) + except AttributeError: + logging.warn("Could not patch modules whitelist. the compiler " + "and parser modules will not work and SSL support " + "is disabled.") + # In SDK 1.6.4, the datastore doesn't save automatically on exit. + # Register a handler to make sure we save. This is important on + # manage.py commands other than 'runserver'. Note that with runserver, + # the datastore is flushed twice. This should be acceptable. + import atexit + if hasattr(dev_appserver, 'TearDownStubs'): + atexit.register(dev_appserver.TearDownStubs) + except ImportError: + pass elif not on_production_server: try: # Restore the real subprocess module. + from google.appengine.tools import dev_appserver from google.appengine.api.mail_stub import subprocess sys.modules['subprocess'] = subprocess # Re-inject the buffer() builtin into the subprocess module. - from google.appengine.tools import dev_appserver subprocess.buffer = dev_appserver.buffer + except ImportError: + # Not required for devappserver2 + pass except Exception, e: logging.warn("Could not add the subprocess module to the " "sandbox: %s" % e) @@ -209,3 +219,4 @@ def setup_project(): while path in sys.path: sys.path.remove(path) sys.path = extra_paths + sys.path + diff --git a/db/stubs.py b/db/stubs.py index 41b7124..112d104 100644 --- a/db/stubs.py +++ b/db/stubs.py @@ -74,16 +74,27 @@ def setup_local_stubs(self, connection): if self.active_stubs == 'local': return from .base import get_datastore_paths - from google.appengine.tools import dev_appserver_main - args = dev_appserver_main.DEFAULT_ARGS.copy() - args.update(get_datastore_paths(connection.settings_dict)) - args.update(connection.settings_dict.get('DEV_APPSERVER_OPTIONS', {})) - log_level = logging.getLogger().getEffectiveLevel() - logging.getLogger().setLevel(logging.WARNING) - from google.appengine.tools import dev_appserver - dev_appserver.SetupStubs('dev~' + appid, **args) - logging.getLogger().setLevel(log_level) - self.active_stubs = 'local' + try: + from google.appengine.tools import dev_appserver_main + args = dev_appserver_main.DEFAULT_ARGS.copy() + args.update(get_datastore_paths(connection.settings_dict)) + args.update(connection.settings_dict.get('DEV_APPSERVER_OPTIONS', {})) + log_level = logging.getLogger().getEffectiveLevel() + logging.getLogger().setLevel(logging.WARNING) + from google.appengine.tools import dev_appserver + dev_appserver.SetupStubs('dev~' + appid, **args) + logging.getLogger().setLevel(log_level) + self.active_stubs = 'local' + except ImportError: + #from google.appengine.tools.devappserver2.devappserver2 import create_command_line_parser + #parser = create_command_line_parser() + #options = parser.parse_args() + log_level = logging.getLogger().getEffectiveLevel() + logging.getLogger().setLevel(logging.WARNING) + #from google.appengine.tools import dev_appserver + #dev_appserver.SetupStubs('dev~' + appid, **args) + #logging.getLogger().setLevel(log_level) + self.active_stubs = 'local' def setup_remote_stubs(self, connection): if self.active_stubs == 'remote': diff --git a/management/commands/runserver.py b/management/commands/runserver.py index 4b5a020..ce2ea15 100644 --- a/management/commands/runserver.py +++ b/management/commands/runserver.py @@ -7,7 +7,17 @@ from django.core.management.commands.runserver import BaseRunserverCommand from django.core.exceptions import ImproperlyConfigured -from google.appengine.tools import dev_appserver_main +try: + from google.appengine.tools import dev_appserver_main +except ImportError: + import os + import google + sys.argv[0] = os.path.join( + os.path.dirname(os.path.dirname(google.__file__)), + "devappserver2.py") + # The following import sets the path for _python_runtime.py from + # sys.argv[0], so we need to hack sys.argv[0] before this import + from google.appengine.tools.devappserver2 import devappserver2 from ...boot import PROJECT_DIR from ...db.base import DatabaseWrapper, get_datastore_paths @@ -102,7 +112,10 @@ def create_parser(self, prog_name, subcommand): parse the arguments to this command. """ # Hack __main__ so --help in dev_appserver_main works OK. - sys.modules['__main__'] = dev_appserver_main + if 'dev_appserver_main' in globals(): + sys.modules['__main__'] = dev_appserver_main + else: + sys.modules['__main__'] = devappserver2 return super(Command, self).create_parser(prog_name, subcommand) def run_from_argv(self, argv): @@ -126,7 +139,7 @@ def run(self, *args, **options): args = [] # Set bind ip/port if specified. if self.addr: - args.extend(['--address', self.addr]) + args.extend(['--host', self.addr]) if self.port: args.extend(['--port', self.port]) @@ -165,10 +178,16 @@ def run(self, *args, **options): break # Process the rest of the options here. - bool_options = [ - 'debug', 'debug_imports', 'clear_datastore', 'require_indexes', - 'high_replication', 'enable_sendmail', 'use_sqlite', - 'allow_skipped_files', 'disable_task_running', ] + if 'dev_appserver_main' in globals(): + bool_options = [ + 'debug', 'debug_imports', 'clear_datastore', 'require_indexes', + 'high_replication', 'enable_sendmail', 'use_sqlite', + 'allow_skipped_files', 'disable_task_running', ] + else: + bool_options = [ + 'debug', 'debug_imports', 'clear_datastore', 'require_indexes', + 'enable_sendmail', 'use_sqlite', + 'allow_skipped_files', 'disable_task_running', ] for opt in bool_options: if options[opt] != False: args.append('--%s' % opt) @@ -195,4 +214,8 @@ def run(self, *args, **options): logging.getLogger().setLevel(logging.INFO) # Append the current working directory to the arguments. - dev_appserver_main.main([self.progname] + args + [PROJECT_DIR]) + if 'dev_appserver_main' in globals(): + dev_appserver_main.main([self.progname] + args + [PROJECT_DIR]) + else: + sys.argv = ['/home/user/google_appengine/devappserver2.py'] + args + [PROJECT_DIR] + devappserver2.main() diff --git a/utils.py b/utils.py index f80b2ab..b830315 100644 --- a/utils.py +++ b/utils.py @@ -10,14 +10,21 @@ appid = get_application_id() else: try: + # Original dev_appserver method from google.appengine.tools import dev_appserver from .boot import PROJECT_DIR appconfig = dev_appserver.LoadAppConfig(PROJECT_DIR, {}, default_partition='dev')[0] appid = appconfig.application.split('~', 1)[-1] except ImportError, e: - raise Exception("Could not get appid. Is your app.yaml file missing? " - "Error was: %s" % e) + try: + from google.appengine.tools.devappserver2 import application_configuration + configuration = application_configuration.ApplicationConfiguration(["app.yaml"]) + appid = configuration.app_id.split('~', 1)[-1] + + except Exception, e: + raise Exception("Could not get appid. Is your app.yaml file missing? " + "Error was: %s" % e) on_production_server = have_appserver and \ not os.environ.get('SERVER_SOFTWARE', '').lower().startswith('devel') From c2293fe42b3be9338c2429ae75c10019aeb3ed3d Mon Sep 17 00:00:00 2001 From: projecteat Date: Fri, 8 Feb 2013 19:18:08 -0500 Subject: [PATCH 35/55] Fix a bug that broke backwards compatibility with 1.7.4 Figured out how to get manage.py test, shell and dumpdata working. --- db/stubs.py | 1 + management/commands/runserver.py | 5 +- test.py | 206 ++++++++++++++++--------------- 3 files changed, 111 insertions(+), 101 deletions(-) diff --git a/db/stubs.py b/db/stubs.py index 112d104..ed7ada9 100644 --- a/db/stubs.py +++ b/db/stubs.py @@ -54,6 +54,7 @@ def activate_test_stubs(self, connection): datastore_opts['consistency_policy'] = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1) self.testbed.activate() + self.testbed.setup_env(True, **{'app_id' : 'dev~' + appid }) self.pre_test_stubs = self.active_stubs self.active_stubs = 'test' self.testbed.init_datastore_v3_stub(**datastore_opts) diff --git a/management/commands/runserver.py b/management/commands/runserver.py index ce2ea15..a238d0b 100644 --- a/management/commands/runserver.py +++ b/management/commands/runserver.py @@ -139,7 +139,10 @@ def run(self, *args, **options): args = [] # Set bind ip/port if specified. if self.addr: - args.extend(['--host', self.addr]) + if 'dev_appserver_main' in globals(): + args.extend(['--address', self.addr]) + else: + args.extend(['--host', self.addr]) if self.port: args.extend(['--port', self.port]) diff --git a/test.py b/test.py index f181320..b2e8676 100644 --- a/test.py +++ b/test.py @@ -12,8 +12,6 @@ from django.test.testcases import _MediaFilesHandler from django.contrib.staticfiles.handlers import StaticFilesHandler -from google.appengine.tools import dev_appserver -from google.appengine.tools import dev_appserver_main from google.appengine.datastore import datastore_stub_util from db.stubs import stub_manager @@ -57,116 +55,124 @@ def addError(self, test, err): test.server_thread.join() super(SyncTextTestResult, self).addError(test, err) -class LiveServerThread(threading.Thread): - """ - Thread for running a live http server while the tests are running. - - This is mostly copied from django.test.testcases.LiveServerThread - It's modified slightly to launch dev_appserver instead of a plain - HTTP server. The shutdown mechanism is slightly different too. +try: + from google.appengine.tools import dev_appserver + from google.appengine.tools import dev_appserver_main - One big problem is that dev_appserver mangles the environment. - It's easy to run into threading issues where the dev_appserver - thread and the main application (test) thread conflict. One common - example is trying to use logging.error(), which will often cause - conflicts since dev_appserver replaces stderr. We use - liveServerLock to avoid these conflicts. + class LiveServerThread(threading.Thread): + """ + Thread for running a live http server while the tests are running. - Your own test code will need to acquire liveServerLock pretty much - every time you're doing something outside of an HTTP request. - """ + This is mostly copied from django.test.testcases.LiveServerThread + It's modified slightly to launch dev_appserver instead of a plain + HTTP server. The shutdown mechanism is slightly different too. - def __init__(self, host, possible_ports, connections_override=None): - self.host = host - self.port = None - self.possible_ports = possible_ports - self.is_ready = threading.Event() - self.error = None - self.connections_override = connections_override - super(LiveServerThread, self).__init__() + One big problem is that dev_appserver mangles the environment. + It's easy to run into threading issues where the dev_appserver + thread and the main application (test) thread conflict. One common + example is trying to use logging.error(), which will often cause + conflicts since dev_appserver replaces stderr. We use + liveServerLock to avoid these conflicts. - def run(self): - """ - Sets up the live server and databases, and then loops over handling - http requests. + Your own test code will need to acquire liveServerLock pretty much + every time you're doing something outside of an HTTP request. """ - def sync_handle_request(self): + def __init__(self, host, possible_ports, connections_override=None): + self.host = host + self.port = None + self.possible_ports = possible_ports + self.is_ready = threading.Event() + self.error = None + self.connections_override = connections_override + super(LiveServerThread, self).__init__() + + def run(self): + """ + Sets up the live server and databases, and then loops over handling + http requests. + """ + + def sync_handle_request(self): + try: + readable, _, _ = select.select([self.socket], [], [], 10) + if readable: + liveServerLock.acquire() + try: + self.original_handle_request() + except Exception, e: + pass + finally: + liveServerLock.release() + except Exception, e: + pass + + if self.connections_override: + from django.db import connections + # Override this thread's database connections with the ones + # provided by the main thread. + for alias, conn in self.connections_override.items(): + connections[alias] = conn try: - readable, _, _ = select.select([self.socket], [], [], 10) - if readable: - liveServerLock.acquire() + # Go through the list of possible ports, hoping that we can find + # one that is free to use for the WSGI server. + for index, port in enumerate(self.possible_ports): try: - self.original_handle_request() - except Exception, e: - pass - finally: - liveServerLock.release() - except Exception, e: - pass - - if self.connections_override: - from django.db import connections - # Override this thread's database connections with the ones - # provided by the main thread. - for alias, conn in self.connections_override.items(): - connections[alias] = conn - try: - # Go through the list of possible ports, hoping that we can find - # one that is free to use for the WSGI server. - for index, port in enumerate(self.possible_ports): - try: - options = dev_appserver_main.DEFAULT_ARGS.copy() - options['disable_task_running'] = True # Prevent launch of task queue thread - dev_appserver.SetupStubs(appid, **options) - - self.httpd = dev_appserver.CreateServer(".", '/_ah/login', port, default_partition="dev") - except socket.error, e: - error_code = e.errno - if (index + 1 < len(self.possible_ports) and - error_code == errno.EADDRINUSE): - # This port is already in use, so we go on and try with - # the next one in the list. - continue + options = dev_appserver_main.DEFAULT_ARGS.copy() + options['disable_task_running'] = True # Prevent launch of task queue thread + dev_appserver.SetupStubs(appid, **options) + + self.httpd = dev_appserver.CreateServer(".", '/_ah/login', port, default_partition="dev") + except socket.error, e: + error_code = e.errno + if (index + 1 < len(self.possible_ports) and + error_code == errno.EADDRINUSE): + # This port is already in use, so we go on and try with + # the next one in the list. + continue + else: + # Either none of the given ports are free or the error + # is something else than "Address already in use". So + # we let that error bubble up to the main thread. + raise + else: - # Either none of the given ports are free or the error - # is something else than "Address already in use". So - # we let that error bubble up to the main thread. - raise + # A free port was found. + self.port = port + break - else: - # A free port was found. - self.port = port - break - - # HACK: The magic happens here. We replace the http request handler - # with our sync'd version - self.httpd.original_handle_request = self.httpd.handle_request - self.httpd.handle_request = types.MethodType(sync_handle_request, self.httpd) - - self.is_ready.set() - self.httpd.serve_forever() - except Exception, e: - self.error = e - try: - self.httpd.server_close() - except Exception, e: - pass - self.is_ready.set() + # HACK: The magic happens here. We replace the http request handler + # with our sync'd version + self.httpd.original_handle_request = self.httpd.handle_request + self.httpd.handle_request = types.MethodType(sync_handle_request, self.httpd) - def join(self, timeout=None): - if hasattr(self, 'httpd'): - # Stop the WSGI server - try: - self.httpd.stop_serving_forever() - # We need to hit the server with one more request to make it quit - connection = httplib.HTTPConnection(self.host, self.port) - connection.request('GET',"/") - connection.close() - #self.httpd.server_close() + self.is_ready.set() + self.httpd.serve_forever() except Exception, e: - pass - super(LiveServerThread, self).join(timeout) + self.error = e + try: + self.httpd.server_close() + except Exception, e: + pass + self.is_ready.set() + + def join(self, timeout=None): + if hasattr(self, 'httpd'): + # Stop the WSGI server + try: + self.httpd.stop_serving_forever() + # We need to hit the server with one more request to make it quit + connection = httplib.HTTPConnection(self.host, self.port) + connection.request('GET',"/") + connection.close() + #self.httpd.server_close() + except Exception, e: + pass + super(LiveServerThread, self).join(timeout) + +except ImportError: + # can't use dev_appserver in devappserver2 + pass # This is copied directly from django.test.testcases class LiveServerTestCase(TransactionTestCase): From 6e94cefae7c27dd8063166571a395d31b64805e2 Mon Sep 17 00:00:00 2001 From: projecteat Date: Fri, 15 Feb 2013 03:11:40 -0500 Subject: [PATCH 36/55] Importing Testbed() on production throws some SSL warnings in the log. --- db/stubs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/stubs.py b/db/stubs.py index ed7ada9..71e78f8 100644 --- a/db/stubs.py +++ b/db/stubs.py @@ -3,8 +3,6 @@ import time from urllib2 import HTTPError, URLError -from google.appengine.ext.testbed import Testbed - from ..boot import PROJECT_DIR from ..utils import appid, have_appserver @@ -29,7 +27,6 @@ def rpc_server_factory(*args, ** kwargs): class StubManager(object): def __init__(self): - self.testbed = Testbed() self.active_stubs = None self.pre_test_stubs = None @@ -40,6 +37,9 @@ def setup_stubs(self, connection): self.setup_local_stubs(connection) def activate_test_stubs(self, connection): + from google.appengine.ext.testbed import Testbed + self.testbed = Testbed() + if self.active_stubs == 'test': return From 6bca0c5230aa9d0ecbaedd54637f21b55d071a6e Mon Sep 17 00:00:00 2001 From: projecteat Date: Tue, 12 Mar 2013 19:59:06 -0400 Subject: [PATCH 37/55] Make 'python manage.py runserver' commands work with either GAE SDK 1.7.5 running dev_appserver, or devappserver2 0.8. Currently we detect between dev_appserver and devappserver2 based on availability of certain modules. In SDK 1.7.5, both dev_appserver and devappserver2 are available, and we default to dev_appserver for backwards compatibility. Does not have a current way to force devappserver2 in this case. --- djangoappengine/boot.py | 13 ++++++++----- djangoappengine/management/commands/runserver.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/djangoappengine/boot.py b/djangoappengine/boot.py index 555ca9b..267451b 100644 --- a/djangoappengine/boot.py +++ b/djangoappengine/boot.py @@ -69,12 +69,15 @@ def setup_env(): sys.path = [ sdk_path ] + sys.path # Then call fix_sys_path from the SDK + from dev_appserver import fix_sys_path try: - from dev_appserver import fix_sys_path - fix_sys_path() - except ImportError: - import devappserver2 - devappserver2.fix_sys_path(devappserver2.DEVAPPSERVER2_PATHS) + # emulate dev_appserver._run_file in devappserver2 + from dev_appserver import _SYS_PATH_ADDITIONS + sys.path = _SYS_PATH_ADDITIONS['_python_runtime.py'] + sys.path + except: + # we're probably on the old dev_appserver + pass + fix_sys_path() setup_project() from .utils import have_appserver diff --git a/djangoappengine/management/commands/runserver.py b/djangoappengine/management/commands/runserver.py index 5d22425..dcf1c51 100644 --- a/djangoappengine/management/commands/runserver.py +++ b/djangoappengine/management/commands/runserver.py @@ -193,7 +193,7 @@ def run(self, *args, **options): else: bool_options = [ 'debug', 'debug_imports', 'clear_datastore', 'require_indexes', - 'enable_sendmail', 'use_sqlite', + 'enable_sendmail', 'allow_skipped_files', 'disable_task_running', ] for opt in bool_options: if options[opt] != False: From 6a472a80c25bc077816c9104b45d5a64f3063273 Mon Sep 17 00:00:00 2001 From: projecteat Date: Sat, 16 Mar 2013 00:05:30 -0400 Subject: [PATCH 38/55] Add url property to BlobStoreFile and BlobstoreUploadedFile. This makes the FileField's ClearableFileInput widget show up properly when there's an existing file in the model. --- djangoappengine/storage.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/djangoappengine/storage.py b/djangoappengine/storage.py index 69e7eb6..e7fef82 100644 --- a/djangoappengine/storage.py +++ b/djangoappengine/storage.py @@ -140,6 +140,13 @@ def file(self): self._file = BlobReader(self.blobstore_info.key()) return self._file + @property + def url(self): + try: + return get_serving_url(self.blobstore_info.key()) + except NotImageError: + return None + class BlobstoreFileUploadHandler(FileUploadHandler): """ @@ -197,3 +204,10 @@ def chunks(self, chunk_size=1024 * 128): def multiple_chunks(self, chunk_size=1024 * 128): return True + + @property + def url(self): + try: + return get_serving_url(self.blobstore_info.key()) + except NotImageError: + return None From ff107ca0e530fa5eb04c03149557bd4ee95a25e5 Mon Sep 17 00:00:00 2001 From: projecteat Date: Tue, 19 Mar 2013 00:46:22 -0400 Subject: [PATCH 39/55] Patch manage.py test to enable file uploads to blobstore. --- djangoappengine/db/stubs.py | 1 + djangoappengine/management/commands/test.py | 50 +++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 djangoappengine/management/commands/test.py diff --git a/djangoappengine/db/stubs.py b/djangoappengine/db/stubs.py index 71e78f8..6d9a2a7 100644 --- a/djangoappengine/db/stubs.py +++ b/djangoappengine/db/stubs.py @@ -65,6 +65,7 @@ def activate_test_stubs(self, connection): 'login_url' : '/_ah/login?continue=%s' }) self.testbed.init_xmpp_stub() self.testbed.init_channel_stub() + self.testbed.init_blobstore_stub(True) def deactivate_test_stubs(self): if self.active_stubs == 'test': diff --git a/djangoappengine/management/commands/test.py b/djangoappengine/management/commands/test.py new file mode 100644 index 0000000..bb9fc87 --- /dev/null +++ b/djangoappengine/management/commands/test.py @@ -0,0 +1,50 @@ +import logging +from optparse import make_option +import sys + +from django.db import connections +from django.core.management.base import BaseCommand +from django.core.management.commands.test import Command as OriginalCommand +from django.core.exceptions import ImproperlyConfigured + +from django.test import client + +original_encode_file = client.encode_file + +def my_encode_file(boundary, key, file): + # encode_file with blobstore support. + # Expecting something like this in the test: + + ''' + from google.appengine.api import files + fn = files.blobstore.create(mime_type="image/jpg", _blobinfo_uploaded_filename="foo.jpg") + with files.open(fn, 'a') as fp: + fp.write("bar") + files.finalize(fn) + blob_key = files.blobstore.get_blob_key(fn) + + with files.open(fn) as fp: + fp.name = "foo.jpg" + fp.blob_key = blob_key + fp.mime_type = "image/jpg" + response = self.client.post('/viewurl', {"fileparam" : fp}) + ''' + + if hasattr(file, "blob_key"): + return [ + '--' + boundary, + 'Content-Type: message/external-body; blob-key=%s; access-type="X-AppEngine-BlobKey"' % file.blob_key, + 'MIME-Version: 1.0', + 'Content-Disposition: form-data; name="%s"; filename="%s"' % (key, file.name), + '', + 'Content-Type: %s' % file.mime_type + ] + else: + return original_encode_file(boundary, key, file) + +class Command(OriginalCommand): + def __init__(self): + # monkey patch client's encode_file with our own + # with blobstore support + client.encode_file = my_encode_file + super(Command, self).__init__() From 0e49f3f0ce881b03dd279a31c0dcc09af84ab18a Mon Sep 17 00:00:00 2001 From: projecteat Date: Tue, 19 Mar 2013 01:40:22 -0400 Subject: [PATCH 40/55] Streamline the last checkin a bit by automatically pulling up the blob content-type, blob-key, and name. --- djangoappengine/management/commands/test.py | 25 ++++++++------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/djangoappengine/management/commands/test.py b/djangoappengine/management/commands/test.py index bb9fc87..c3372df 100644 --- a/djangoappengine/management/commands/test.py +++ b/djangoappengine/management/commands/test.py @@ -1,14 +1,10 @@ -import logging -from optparse import make_option -import sys - -from django.db import connections -from django.core.management.base import BaseCommand from django.core.management.commands.test import Command as OriginalCommand -from django.core.exceptions import ImproperlyConfigured from django.test import client +from google.appengine.api import files +from google.appengine.ext.blobstore import BlobInfo + original_encode_file = client.encode_file def my_encode_file(boundary, key, file): @@ -21,23 +17,20 @@ def my_encode_file(boundary, key, file): with files.open(fn, 'a') as fp: fp.write("bar") files.finalize(fn) - blob_key = files.blobstore.get_blob_key(fn) with files.open(fn) as fp: - fp.name = "foo.jpg" - fp.blob_key = blob_key - fp.mime_type = "image/jpg" response = self.client.post('/viewurl', {"fileparam" : fp}) ''' - - if hasattr(file, "blob_key"): + if hasattr(file, "_filename"): + blob_key = files.blobstore.get_blob_key(file._filename) + blobinfo = BlobInfo.get(blob_key) return [ '--' + boundary, - 'Content-Type: message/external-body; blob-key=%s; access-type="X-AppEngine-BlobKey"' % file.blob_key, + 'Content-Type: message/external-body; blob-key=%s; access-type="X-AppEngine-BlobKey"' % blob_key, 'MIME-Version: 1.0', - 'Content-Disposition: form-data; name="%s"; filename="%s"' % (key, file.name), + 'Content-Disposition: form-data; name="%s"; filename="%s"' % (key, blobinfo.filename), '', - 'Content-Type: %s' % file.mime_type + 'Content-Type: %s' % blobinfo.content_type ] else: return original_encode_file(boundary, key, file) From ec08bb8730f038a9bfd7959b24c0c671db41b540 Mon Sep 17 00:00:00 2001 From: projecteat Date: Tue, 19 Mar 2013 15:50:45 -0400 Subject: [PATCH 41/55] Add testing of File fields that store BlobstoreFiles --- djangoappengine/tests/field_db_conversion.py | 16 ++++++++++++++-- djangoappengine/tests/testmodels.py | 8 ++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/djangoappengine/tests/field_db_conversion.py b/djangoappengine/tests/field_db_conversion.py index 29b91e9..9ed98d6 100644 --- a/djangoappengine/tests/field_db_conversion.py +++ b/djangoappengine/tests/field_db_conversion.py @@ -1,13 +1,17 @@ import datetime from django.test import TestCase +from django.db.models.fields.files import FieldFile from google.appengine.api.datastore import Get from google.appengine.api.datastore_types import Text, Category, Email, \ Link, PhoneNumber, PostalAddress, Text, Blob, ByteString, GeoPt, IM, \ Key, Rating, BlobKey +from google.appengine.api import files +from django.core.files.base import ContentFile from .testmodels import FieldsWithoutOptionsModel +from djangoappengine.storage import BlobstoreFile, BlobstoreStorage # TODO: Add field conversions for ForeignKeys? @@ -16,12 +20,18 @@ class FieldDBConversionTest(TestCase): def test_db_conversion(self): actual_datetime = datetime.datetime.now() + + bs = BlobstoreStorage() + fn = bs.save("foo.txt", ContentFile("test")) + bf = BlobstoreFile(fn, 'a', bs) + entity = FieldsWithoutOptionsModel( datetime=actual_datetime, date=actual_datetime.date(), time=actual_datetime.time(), floating_point=5.97, boolean=True, null_boolean=False, text='Hallo', email='hallo@hallo.com', comma_seperated_integer='5,4,3,2', ip_address='194.167.1.1', slug='you slugy slut :)', + file=bf, file_path=bf.name, url='http://www.scholardocs.com', long_text=1000 * 'A', indexed_text='hello', integer=-400, small_integer=-4, positive_integer=400, @@ -38,6 +48,7 @@ def test_db_conversion(self): for name, types in [('long_text', Text), ('indexed_text', unicode), ('text', unicode), ('ip_address', unicode), ('slug', unicode), + ('file', unicode), ('file_path', unicode), ('email', unicode), ('comma_seperated_integer', unicode), ('url', unicode), ('time', datetime.datetime), ('datetime', datetime.datetime), ('date', datetime.datetime), @@ -49,7 +60,7 @@ def test_db_conversion(self): column = opts.get_field_by_name(name)[0].column if not isinstance(types, (list, tuple)): types = (types, ) - self.assertTrue(type(gae_entity[column]) in types) + self.assertTrue(type(gae_entity[column]) in types, column) # Get the model instance and check if the fields convert back # to the right types. @@ -59,6 +70,7 @@ def test_db_conversion(self): ('indexed_text', unicode), ('text', unicode), ('ip_address', unicode), ('slug', unicode), + ('file', FieldFile), ('file_path', unicode), ('email', unicode), ('comma_seperated_integer', unicode), ('url', unicode), ('datetime', datetime.datetime), ('date', datetime.date), ('time', datetime.time), @@ -69,4 +81,4 @@ def test_db_conversion(self): ('positive_small_integer', (int, long))]: if not isinstance(types, (list, tuple)): types = (types, ) - self.assertTrue(type(getattr(model, name)) in types) + self.assertTrue(type(getattr(model, name)) in types, name) diff --git a/djangoappengine/tests/testmodels.py b/djangoappengine/tests/testmodels.py index 0b4bdc4..5e5b0fa 100644 --- a/djangoappengine/tests/testmodels.py +++ b/djangoappengine/tests/testmodels.py @@ -29,8 +29,8 @@ class FieldsWithoutOptionsModel(models.Model): ip_address = models.IPAddressField() slug = models.SlugField() url = models.URLField() -# file = models.FileField() -# file_path = models.FilePathField() + file = models.FileField() + file_path = models.FilePathField() long_text = models.TextField() indexed_text = models.TextField() integer = models.IntegerField() @@ -67,8 +67,8 @@ class FieldsWithOptionsModel(models.Model): ip_address = models.IPAddressField(default='192.168.0.2') slug = models.SlugField(default='GAGAA', null=True) url = models.URLField(default='http://www.scholardocs.com') -# file = FileField() -# file_path = FilePathField() + # file = models.FileField() + # file_path = models.FilePathField() long_text = models.TextField(default=1000 * 'A') integer = models.IntegerField(default=100) small_integer = models.SmallIntegerField(default=-5) From fdc3e1795461b1e2663d5e5662008eab4c637374 Mon Sep 17 00:00:00 2001 From: projecteat Date: Tue, 19 Mar 2013 17:29:30 -0400 Subject: [PATCH 42/55] Update to work on SDK 1.7.6 Force use of devappserver2 by putting a file in the root of your project called force_devappserver.py The file should contain a single variable: devappsever_ver = 1 or 2 manage.py runserver, test and remote shell commands work on both devappserver1/2 manage.py shell works on devappserver1 but not devappserver2. Other commands have not been tested. --- djangoappengine/boot.py | 40 +++++++++---------- djangoappengine/db/stubs.py | 8 ++-- .../management/commands/runserver.py | 14 ++++--- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/djangoappengine/boot.py b/djangoappengine/boot.py index 267451b..04319e5 100644 --- a/djangoappengine/boot.py +++ b/djangoappengine/boot.py @@ -2,6 +2,13 @@ import os import sys +global devappserver_ver +try: + import force_devappserver + devappserver_ver = force_devappserver.devappserver_ver +except: + devappserver_ver = 1 + def find_project_dir(): """ Go through the path, and look for manage.py @@ -70,13 +77,10 @@ def setup_env(): # Then call fix_sys_path from the SDK from dev_appserver import fix_sys_path - try: + if devappserver_ver == 2: # emulate dev_appserver._run_file in devappserver2 from dev_appserver import _SYS_PATH_ADDITIONS sys.path = _SYS_PATH_ADDITIONS['_python_runtime.py'] + sys.path - except: - # we're probably on the old dev_appserver - pass fix_sys_path() setup_project() @@ -153,7 +157,7 @@ def setup_project(): # enable https connections (seem to be broken on Windows because # the _ssl module is disallowed). if not have_appserver: - try: + if devappserver_ver == 1: from google.appengine.tools import dev_appserver try: # Backup os.environ. It gets overwritten by the @@ -185,22 +189,18 @@ def setup_project(): import atexit if hasattr(dev_appserver, 'TearDownStubs'): atexit.register(dev_appserver.TearDownStubs) - except ImportError: - pass elif not on_production_server: - try: - # Restore the real subprocess module. - from google.appengine.tools import dev_appserver - from google.appengine.api.mail_stub import subprocess - sys.modules['subprocess'] = subprocess - # Re-inject the buffer() builtin into the subprocess module. - subprocess.buffer = dev_appserver.buffer - except ImportError: - # Not required for devappserver2 - pass - except Exception, e: - logging.warn("Could not add the subprocess module to the " - "sandbox: %s" % e) + if devappserver_ver == 1: + try: + # Restore the real subprocess module. + from google.appengine.tools import dev_appserver + from google.appengine.api.mail_stub import subprocess + sys.modules['subprocess'] = subprocess + # Re-inject the buffer() builtin into the subprocess module. + subprocess.buffer = dev_appserver.buffer + except Exception, e: + logging.warn("Could not add the subprocess module to the " + "sandbox: %s" % e) os.environ.update(env_ext) diff --git a/djangoappengine/db/stubs.py b/djangoappengine/db/stubs.py index 6d9a2a7..58c5f3a 100644 --- a/djangoappengine/db/stubs.py +++ b/djangoappengine/db/stubs.py @@ -3,7 +3,7 @@ import time from urllib2 import HTTPError, URLError -from ..boot import PROJECT_DIR +from ..boot import PROJECT_DIR, devappserver_ver from ..utils import appid, have_appserver @@ -65,7 +65,9 @@ def activate_test_stubs(self, connection): 'login_url' : '/_ah/login?continue=%s' }) self.testbed.init_xmpp_stub() self.testbed.init_channel_stub() + self.testbed.init_files_stub(True) self.testbed.init_blobstore_stub(True) + self.testbed.init_images_stub(True) def deactivate_test_stubs(self): if self.active_stubs == 'test': @@ -76,7 +78,7 @@ def setup_local_stubs(self, connection): if self.active_stubs == 'local': return from .base import get_datastore_paths - try: + if devappserver_ver == 1: from google.appengine.tools import dev_appserver_main args = dev_appserver_main.DEFAULT_ARGS.copy() args.update(get_datastore_paths(connection.settings_dict)) @@ -87,7 +89,7 @@ def setup_local_stubs(self, connection): dev_appserver.SetupStubs('dev~' + appid, **args) logging.getLogger().setLevel(log_level) self.active_stubs = 'local' - except ImportError: + elif devappserver_ver == 2: #from google.appengine.tools.devappserver2.devappserver2 import create_command_line_parser #parser = create_command_line_parser() #options = parser.parse_args() diff --git a/djangoappengine/management/commands/runserver.py b/djangoappengine/management/commands/runserver.py index dcf1c51..8fb621c 100644 --- a/djangoappengine/management/commands/runserver.py +++ b/djangoappengine/management/commands/runserver.py @@ -7,9 +7,11 @@ from django.core.management.commands.runserver import BaseRunserverCommand from django.core.exceptions import ImproperlyConfigured -try: +from ...boot import devappserver_ver + +if devappserver_ver == 1: from google.appengine.tools import dev_appserver_main -except ImportError: +elif devappserver_ver == 2: import os import google sys.argv[0] = os.path.join( @@ -116,7 +118,7 @@ def create_parser(self, prog_name, subcommand): parse the arguments to this command. """ # Hack __main__ so --help in dev_appserver_main works OK. - if 'dev_appserver_main' in globals(): + if devappserver_ver == 1: sys.modules['__main__'] = dev_appserver_main else: sys.modules['__main__'] = devappserver2 @@ -143,7 +145,7 @@ def run(self, *args, **options): args = [] # Set bind ip/port if specified. if self.addr: - if 'dev_appserver_main' in globals(): + if devappserver_ver == 1: args.extend(['--address', self.addr]) else: args.extend(['--host', self.addr]) @@ -185,7 +187,7 @@ def run(self, *args, **options): break # Process the rest of the options here. - if 'dev_appserver_main' in globals(): + if devappserver_ver == 1: bool_options = [ 'debug', 'debug_imports', 'clear_datastore', 'require_indexes', 'high_replication', 'enable_sendmail', 'use_sqlite', @@ -221,7 +223,7 @@ def run(self, *args, **options): logging.getLogger().setLevel(logging.INFO) # Append the current working directory to the arguments. - if 'dev_appserver_main' in globals(): + if devappserver_ver == 1: dev_appserver_main.main([self.progname] + args + [PROJECT_DIR]) else: sys.argv = ['/home/user/google_appengine/devappserver2.py'] + args + [PROJECT_DIR] From 2a09904d9ec0f3bd53e4d67f03fd7ac27a2af728 Mon Sep 17 00:00:00 2001 From: projecteat Date: Tue, 19 Mar 2013 23:11:52 -0400 Subject: [PATCH 43/55] Make manage.py shell work for datastore accesses on devappserver2 --- djangoappengine/management/commands/shell.py | 75 ++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 djangoappengine/management/commands/shell.py diff --git a/djangoappengine/management/commands/shell.py b/djangoappengine/management/commands/shell.py new file mode 100644 index 0000000..6e9b507 --- /dev/null +++ b/djangoappengine/management/commands/shell.py @@ -0,0 +1,75 @@ +from django.core.management.commands.shell import Command as ShellCommand +from ...boot import devappserver_ver + +class Command(ShellCommand): + def __init__(self): + super(Command, self).__init__() + if devappserver_ver == 2: + # This setup is usually done in devappserver2.DevelopmentServer + # We don't want to run the full server when we just need the shell + # Just use the code to setup the datastore stubs. + import os + import time + from google.appengine.datastore import datastore_stub_util + from google.appengine.tools.devappserver2 import api_server + from google.appengine.tools.devappserver2 import application_configuration + + # Mimic google.appengine.tools.devappserver2 + os.environ['TZ'] = 'UTC' + if hasattr(time, 'tzset'): + # time.tzet() should be called on Unix, but doesn't exist on Windows. + time.tzset() + + from google.appengine.tools.devappserver2.devappserver2 import PARSER, _get_storage_path, _setup_environ + options = PARSER.parse_args(['--admin_port', '0', + '--port', '0', + '--datastore_path', '.gaedata/datastore', + '--logs_path', ':memory', + '--skip_sdk_update_check', "yes", + "."]) + + configuration = application_configuration.ApplicationConfiguration( + options.yaml_files) + + _setup_environ(configuration.app_id) + + storage_path = _get_storage_path(options.storage_path, configuration.app_id) + datastore_path = options.datastore_path or os.path.join(storage_path, + 'datastore.db') + logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') + + search_index_path = options.search_indexes_path or os.path.join( + storage_path, 'search_indexes') + + prospective_search_path = options.prospective_search_path or os.path.join( + storage_path, 'prospective-search') + + blobstore_path = options.blobstore_path or os.path.join(storage_path, + 'blobs') + + api_server.setup_stubs( + request_data='', + app_id=configuration.app_id, + application_root=configuration.servers[0].application_root, + # The "trusted" flag is only relevant for Google administrative + # applications. + trusted=getattr(options, 'trusted', False), + blobstore_path=blobstore_path, + datastore_path=datastore_path, + datastore_consistency=datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0), + datastore_require_indexes=options.require_indexes, + datastore_auto_id_policy=options.auto_id_policy, + images_host_prefix='', + logs_path=logs_path, + mail_smtp_host=options.smtp_host, + mail_smtp_port=options.smtp_port, + mail_smtp_user=options.smtp_user, + mail_smtp_password=options.smtp_password, + mail_enable_sendmail=options.enable_sendmail, + mail_show_mail_body=options.show_mail_body, + matcher_prospective_search_path=prospective_search_path, + search_index_path=search_index_path, + taskqueue_auto_run_tasks=options.enable_task_running, + taskqueue_default_http_server='%s'%options.host, + user_login_url='', + user_logout_url='') From 49931042b55c20991a38f57732d0fd3243637684 Mon Sep 17 00:00:00 2001 From: projecteat Date: Wed, 20 Mar 2013 12:04:32 -0400 Subject: [PATCH 44/55] Add support for devappserver2 to LiveServerTestCase. Can currently only run one test at a time, since server sometimes fails to shut down. --- djangoappengine/test.py | 284 ++++++++++++++++++++++++++++------------ 1 file changed, 197 insertions(+), 87 deletions(-) diff --git a/djangoappengine/test.py b/djangoappengine/test.py index b2e8676..b4131da 100644 --- a/djangoappengine/test.py +++ b/djangoappengine/test.py @@ -5,12 +5,8 @@ import httplib import socket import select -from django.core.handlers.wsgi import WSGIHandler -from django.core.servers.basehttp import WSGIServerException from django.db import connections from django.test import TestCase, TransactionTestCase -from django.test.testcases import _MediaFilesHandler -from django.contrib.staticfiles.handlers import StaticFilesHandler from google.appengine.datastore import datastore_stub_util @@ -18,6 +14,8 @@ from utils import appid from unittest.runner import TextTestResult +from boot import devappserver_ver + class GAETestCase(TestCase): ''' This base class configures the dev_appserver datastore to test for eventual consistency behavior. @@ -55,7 +53,7 @@ def addError(self, test, err): test.server_thread.join() super(SyncTextTestResult, self).addError(test, err) -try: +if devappserver_ver == 1: from google.appengine.tools import dev_appserver from google.appengine.tools import dev_appserver_main @@ -170,88 +168,200 @@ def join(self, timeout=None): pass super(LiveServerThread, self).join(timeout) -except ImportError: - # can't use dev_appserver in devappserver2 - pass - -# This is copied directly from django.test.testcases -class LiveServerTestCase(TransactionTestCase): - """ - Does basically the same as TransactionTestCase but also launches a live - http server in a separate thread so that the tests may use another testing - framework, such as Selenium for example, instead of the built-in dummy - client. - - Note that it inherits from TransactionTestCase instead of TestCase because - the threads do not share the same transactions (unless if using in-memory - sqlite) and each thread needs to commit all their transactions so that the - other thread can see the changes. - - Be careful that almost everything you do needs to be synchronized against - the liveServerLock (which you can easily reference as this.lock() - """ - lock = liveServerLock - - @property - def live_server_url(self): - return 'http://%s:%s' % ( - self.server_thread.host, self.server_thread.port) - def _pre_setup(self): - connections_override = {} - for conn in connections.all(): - # If using in-memory sqlite databases, pass the connections to - # the server thread. - if (conn.settings_dict['ENGINE'] == 'django.db.backends.sqlite3' - and conn.settings_dict['NAME'] == ':memory:'): - # Explicitly enable thread-shareability for this connection - conn.allow_thread_sharing = True - connections_override[conn.alias] = conn - - # Launch the live server's thread - specified_address = os.environ.get( - 'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081-8089') - - # The specified ports may be of the form '8000-8010,8080,9200-9300' - # i.e. a comma-separated list of ports or ranges gg ports, so we break - # it down into a detailed list of all possible ports. - possible_ports = [] - try: - host, port_ranges = specified_address.split(':') - for port_range in port_ranges.split(','): - # A port range can be of either form: '8000' or '8000-8010'. - extremes = map(int, port_range.split('-')) - assert len(extremes) in [1, 2] - if len(extremes) == 1: - # Port range of the form '8000' - possible_ports.append(extremes[0]) - else: - # Port range of the form '8000-8010' - for port in range(extremes[0], extremes[1] + 1): - possible_ports.append(port) - except Exception: - raise ImproperlyConfigured('Invalid address ("%s") for live ' - 'server.' % specified_address) - - self.server_thread = LiveServerThread( - host, possible_ports, connections_override) - self.server_thread.daemon = True - self.server_thread.start() - - # Wait for the live server to be ready - self.server_thread.is_ready.wait() - if self.server_thread.error: - raise self.server_thread.error - - liveServerLock.acquire() # Lock while we load fixtures - super(LiveServerTestCase, self)._pre_setup() - liveServerLock.release() + # This is copied directly from django.test.testcases + class LiveServerTestCase(TransactionTestCase): + """ + Does basically the same as TransactionTestCase but also launches a live + http server in a separate thread so that the tests may use another testing + framework, such as Selenium for example, instead of the built-in dummy + client. + + Note that it inherits from TransactionTestCase instead of TestCase because + the threads do not share the same transactions (unless if using in-memory + sqlite) and each thread needs to commit all their transactions so that the + other thread can see the changes. + + Be careful that almost everything you do needs to be synchronized against + the liveServerLock (which you can easily reference as this.lock() + """ + lock = liveServerLock + + @property + def live_server_url(self): + return 'http://%s:%s' % ( + self.server_thread.host, self.server_thread.port) + + def _pre_setup(self): + connections_override = {} + for conn in connections.all(): + # If using in-memory sqlite databases, pass the connections to + # the server thread. + if (conn.settings_dict['ENGINE'] == 'django.db.backends.sqlite3' + and conn.settings_dict['NAME'] == ':memory:'): + # Explicitly enable thread-shareability for this connection + conn.allow_thread_sharing = True + connections_override[conn.alias] = conn + + # Launch the live server's thread + specified_address = os.environ.get( + 'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081-8089') + + # The specified ports may be of the form '8000-8010,8080,9200-9300' + # i.e. a comma-separated list of ports or ranges gg ports, so we break + # it down into a detailed list of all possible ports. + possible_ports = [] + try: + host, port_ranges = specified_address.split(':') + for port_range in port_ranges.split(','): + # A port range can be of either form: '8000' or '8000-8010'. + extremes = map(int, port_range.split('-')) + assert len(extremes) in [1, 2] + if len(extremes) == 1: + # Port range of the form '8000' + possible_ports.append(extremes[0]) + else: + # Port range of the form '8000-8010' + for port in range(extremes[0], extremes[1] + 1): + possible_ports.append(port) + except Exception: + raise ImproperlyConfigured('Invalid address ("%s") for live ' + 'server.' % specified_address) + + self.server_thread = LiveServerThread( + host, possible_ports, connections_override) + self.server_thread.daemon = True + self.server_thread.start() + + # Wait for the live server to be ready + self.server_thread.is_ready.wait() + if self.server_thread.error: + raise self.server_thread.error + + liveServerLock.acquire() # Lock while we load fixtures + super(LiveServerTestCase, self)._pre_setup() + liveServerLock.release() + + def _post_teardown(self): + # There may not be a 'server_thread' attribute if setUpClass() for some + # reasons has raised an exception. + if hasattr(self, 'server_thread'): + # Terminate the live server's thread + self.server_thread.join() + super(LiveServerTestCase, self)._post_teardown() + +else: # devappserver2 + import sys + from django.db import DEFAULT_DB_ALIAS + from django.core.management import call_command + import dev_appserver + from google.appengine.api import apiproxy_stub_map + from google.appengine.tools.devappserver2 import devappserver2 + from google.appengine.tools.devappserver2 import python_runtime + from google.appengine.tools.devappserver2 import shutdown + + sys.path[1:1] = dev_appserver._DEVAPPSERVER2_PATHS + + class LiveServerTestCase(TransactionTestCase): + """ + Does basically the same as TransactionTestCase but also launches a live + http server in a separate thread so that the tests may use another testing + framework, such as Selenium for example, instead of the built-in dummy + client. + + Note that it inherits from TransactionTestCase instead of TestCase because + the threads do not share the same transactions (unless if using in-memory + sqlite) and each thread needs to commit all their transactions so that the + other thread can see the changes. + + Be careful that almost everything you do needs to be synchronized against + the liveServerLock (which you can easily reference as this.lock() + """ + lock = liveServerLock - def _post_teardown(self): - # There may not be a 'server_thread' attribute if setUpClass() for some - # reasons has raised an exception. - if hasattr(self, 'server_thread'): - # Terminate the live server's thread - self.server_thread.join() - super(LiveServerTestCase, self)._post_teardown() + @property + def live_server_url(self): + return 'http://%s:%s' % ( + 'localhost', self.port) + + def _pre_setup(self): + apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() + self.server = None + + # Launch the live server's thread + specified_address = os.environ.get( + 'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081-8089') + + # The specified ports may be of the form '8000-8010,8080,9200-9300' + # i.e. a comma-separated list of ports or ranges gg ports, so we break + # it down into a detailed list of all possible ports. + possible_ports = [] + try: + host, port_ranges = specified_address.split(':') + for port_range in port_ranges.split(','): + # A port range can be of either form: '8000' or '8000-8010'. + extremes = map(int, port_range.split('-')) + assert len(extremes) in [1, 2] + if len(extremes) == 1: + # Port range of the form '8000' + possible_ports.append(extremes[0]) + else: + # Port range of the form '8000-8010' + for port in range(extremes[0], extremes[1] + 1): + possible_ports.append(port) + except Exception: + raise ImproperlyConfigured('Invalid address ("%s") for live ' + 'server.' % specified_address) + + python_runtime._RUNTIME_ARGS = [ + sys.executable, + os.path.join( + os.path.dirname(dev_appserver.__file__), '_python_runtime.py')] + options = devappserver2.PARSER.parse_args([ + '--admin_port', '0', + '--port', '%s' % possible_ports[0], + '--datastore_path', ':memory:', + '--logs_path', ':memory:', + '--skip_sdk_update_check', + '--', + '.']) + self.port = possible_ports[0] + self.server = devappserver2.DevelopmentServer() + self.server.start(options) + + # Wait for the live server to be ready + # How do we sync this? + + liveServerLock.acquire() # Lock while we load fixtures + super(LiveServerTestCase, self)._pre_setup() + liveServerLock.release() + + def _fixture_setup(self): + # If the test case has a multi_db=True flag, flush all databases. + # Otherwise, just flush default. + if getattr(self, 'multi_db', False): + databases = connections + else: + databases = [DEFAULT_DB_ALIAS] + try: + for db in databases: + # call_command('flush', verbosity=0, interactive=False, database=db) + + if hasattr(self, 'fixtures'): + # We have to use this slightly awkward syntax due to the fact + # that we're using *args and **kwargs together. + call_command('loaddata', *self.fixtures, + **{'verbosity': 0, 'database': db}) + except Exception, e: + pass + + def _post_teardown(self): + if self.server: + print 'Stopping server' + self.server.stop() + print 'Server stopped' + import eat + eat.gaebp() + + super(LiveServerTestCase, self)._post_teardown() From 29ae1a1f43a72ff8208b1126bc0e90f20df1a00e Mon Sep 17 00:00:00 2001 From: projecteat Date: Wed, 20 Mar 2013 12:04:51 -0400 Subject: [PATCH 45/55] Clean up breakpoint. --- djangoappengine/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/djangoappengine/test.py b/djangoappengine/test.py index b4131da..2370368 100644 --- a/djangoappengine/test.py +++ b/djangoappengine/test.py @@ -360,8 +360,6 @@ def _post_teardown(self): print 'Stopping server' self.server.stop() print 'Server stopped' - import eat - eat.gaebp() super(LiveServerTestCase, self)._post_teardown() From ba8122b961522c87967274423c1e4b50909c5657 Mon Sep 17 00:00:00 2001 From: projecteat Date: Thu, 21 Mar 2013 23:25:51 -0400 Subject: [PATCH 46/55] Add support for auto_id_policy to the command line. --- djangoappengine/management/commands/runserver.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/djangoappengine/management/commands/runserver.py b/djangoappengine/management/commands/runserver.py index 8fb621c..6215258 100644 --- a/djangoappengine/management/commands/runserver.py +++ b/djangoappengine/management/commands/runserver.py @@ -36,6 +36,10 @@ class Command(BaseRunserverCommand): """ option_list = BaseCommand.option_list + ( + make_option( + '--auto_id_policy', + help="Dictate how automatic IDs are assigned by the datastore " \ + "stub. 'sequential' or 'scattered'."), make_option( '--debug', action='store_true', default=False, help="Prints verbose debugging messages to the console while " \ @@ -203,7 +207,7 @@ def run(self, *args, **options): str_options = [ 'datastore_path', 'blobstore_path', 'history_path', 'login_url', 'smtp_host', - 'smtp_port', 'smtp_user', 'smtp_password', ] + 'smtp_port', 'smtp_user', 'smtp_password', 'auto_id_policy'] for opt in str_options: if options.get(opt, None) != None: args.extend(['--%s' % opt, options[opt]]) From d607801a4c4760b75dba583a08309e541369260f Mon Sep 17 00:00:00 2001 From: projecteat Date: Sat, 23 Mar 2013 17:06:15 -0400 Subject: [PATCH 47/55] Make sure we set up datastore stubs properly in stubs.py, this makes commands like manage.py shell/dumpdat work. We need to patch runserver a bit so that it resets the stubs, otherwise we get an exception in devappserver2. --- djangoappengine/db/stubs.py | 37 ++++----- .../management/commands/runserver.py | 8 +- djangoappengine/management/commands/shell.py | 75 ------------------- djangoappengine/utils.py | 5 +- 4 files changed, 26 insertions(+), 99 deletions(-) delete mode 100644 djangoappengine/management/commands/shell.py diff --git a/djangoappengine/db/stubs.py b/djangoappengine/db/stubs.py index 58c5f3a..e38b11a 100644 --- a/djangoappengine/db/stubs.py +++ b/djangoappengine/db/stubs.py @@ -78,27 +78,22 @@ def setup_local_stubs(self, connection): if self.active_stubs == 'local': return from .base import get_datastore_paths - if devappserver_ver == 1: - from google.appengine.tools import dev_appserver_main - args = dev_appserver_main.DEFAULT_ARGS.copy() - args.update(get_datastore_paths(connection.settings_dict)) - args.update(connection.settings_dict.get('DEV_APPSERVER_OPTIONS', {})) - log_level = logging.getLogger().getEffectiveLevel() - logging.getLogger().setLevel(logging.WARNING) - from google.appengine.tools import dev_appserver - dev_appserver.SetupStubs('dev~' + appid, **args) - logging.getLogger().setLevel(log_level) - self.active_stubs = 'local' - elif devappserver_ver == 2: - #from google.appengine.tools.devappserver2.devappserver2 import create_command_line_parser - #parser = create_command_line_parser() - #options = parser.parse_args() - log_level = logging.getLogger().getEffectiveLevel() - logging.getLogger().setLevel(logging.WARNING) - #from google.appengine.tools import dev_appserver - #dev_appserver.SetupStubs('dev~' + appid, **args) - #logging.getLogger().setLevel(log_level) - self.active_stubs = 'local' + from google.appengine.tools import dev_appserver_main + args = dev_appserver_main.DEFAULT_ARGS.copy() + args.update(get_datastore_paths(connection.settings_dict)) + args.update(connection.settings_dict.get('DEV_APPSERVER_OPTIONS', {})) + log_level = logging.getLogger().getEffectiveLevel() + logging.getLogger().setLevel(logging.WARNING) + from google.appengine.tools import dev_appserver + dev_appserver.SetupStubs('dev~' + appid, **args) + logging.getLogger().setLevel(log_level) + self.active_stubs = 'local' + if devappserver_ver == 2: + # Mimic google.appengine.tools.devappserver2 + os.environ['TZ'] = 'UTC' + if hasattr(time, 'tzset'): + # time.tzet() should be called on Unix, but doesn't exist on Windows. + time.tzset() def setup_remote_stubs(self, connection): if self.active_stubs == 'remote': diff --git a/djangoappengine/management/commands/runserver.py b/djangoappengine/management/commands/runserver.py index 6215258..4765074 100644 --- a/djangoappengine/management/commands/runserver.py +++ b/djangoappengine/management/commands/runserver.py @@ -5,7 +5,6 @@ from django.db import connections from django.core.management.base import BaseCommand from django.core.management.commands.runserver import BaseRunserverCommand -from django.core.exceptions import ImproperlyConfigured from ...boot import devappserver_ver @@ -230,5 +229,12 @@ def run(self, *args, **options): if devappserver_ver == 1: dev_appserver_main.main([self.progname] + args + [PROJECT_DIR]) else: + from google.appengine.api import apiproxy_stub_map + from google.appengine.tools.devappserver2 import devappserver2 + + # Environment is set in djangoappengine.stubs.setup_local_stubs() + # We need to do this effectively reset the stubs. + apiproxy_stub_map.apiproxy = apiproxy_stub_map.GetDefaultAPIProxy() + sys.argv = ['/home/user/google_appengine/devappserver2.py'] + args + [PROJECT_DIR] devappserver2.main() diff --git a/djangoappengine/management/commands/shell.py b/djangoappengine/management/commands/shell.py deleted file mode 100644 index 6e9b507..0000000 --- a/djangoappengine/management/commands/shell.py +++ /dev/null @@ -1,75 +0,0 @@ -from django.core.management.commands.shell import Command as ShellCommand -from ...boot import devappserver_ver - -class Command(ShellCommand): - def __init__(self): - super(Command, self).__init__() - if devappserver_ver == 2: - # This setup is usually done in devappserver2.DevelopmentServer - # We don't want to run the full server when we just need the shell - # Just use the code to setup the datastore stubs. - import os - import time - from google.appengine.datastore import datastore_stub_util - from google.appengine.tools.devappserver2 import api_server - from google.appengine.tools.devappserver2 import application_configuration - - # Mimic google.appengine.tools.devappserver2 - os.environ['TZ'] = 'UTC' - if hasattr(time, 'tzset'): - # time.tzet() should be called on Unix, but doesn't exist on Windows. - time.tzset() - - from google.appengine.tools.devappserver2.devappserver2 import PARSER, _get_storage_path, _setup_environ - options = PARSER.parse_args(['--admin_port', '0', - '--port', '0', - '--datastore_path', '.gaedata/datastore', - '--logs_path', ':memory', - '--skip_sdk_update_check', "yes", - "."]) - - configuration = application_configuration.ApplicationConfiguration( - options.yaml_files) - - _setup_environ(configuration.app_id) - - storage_path = _get_storage_path(options.storage_path, configuration.app_id) - datastore_path = options.datastore_path or os.path.join(storage_path, - 'datastore.db') - logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') - - search_index_path = options.search_indexes_path or os.path.join( - storage_path, 'search_indexes') - - prospective_search_path = options.prospective_search_path or os.path.join( - storage_path, 'prospective-search') - - blobstore_path = options.blobstore_path or os.path.join(storage_path, - 'blobs') - - api_server.setup_stubs( - request_data='', - app_id=configuration.app_id, - application_root=configuration.servers[0].application_root, - # The "trusted" flag is only relevant for Google administrative - # applications. - trusted=getattr(options, 'trusted', False), - blobstore_path=blobstore_path, - datastore_path=datastore_path, - datastore_consistency=datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0), - datastore_require_indexes=options.require_indexes, - datastore_auto_id_policy=options.auto_id_policy, - images_host_prefix='', - logs_path=logs_path, - mail_smtp_host=options.smtp_host, - mail_smtp_port=options.smtp_port, - mail_smtp_user=options.smtp_user, - mail_smtp_password=options.smtp_password, - mail_enable_sendmail=options.enable_sendmail, - mail_show_mail_body=options.show_mail_body, - matcher_prospective_search_path=prospective_search_path, - search_index_path=search_index_path, - taskqueue_auto_run_tasks=options.enable_task_running, - taskqueue_default_http_server='%s'%options.host, - user_login_url='', - user_logout_url='') diff --git a/djangoappengine/utils.py b/djangoappengine/utils.py index b830315..77f4f68 100644 --- a/djangoappengine/utils.py +++ b/djangoappengine/utils.py @@ -9,14 +9,15 @@ if have_appserver: appid = get_application_id() else: - try: + from boot import devappserver_ver + if devappserver_ver == 1: # Original dev_appserver method from google.appengine.tools import dev_appserver from .boot import PROJECT_DIR appconfig = dev_appserver.LoadAppConfig(PROJECT_DIR, {}, default_partition='dev')[0] appid = appconfig.application.split('~', 1)[-1] - except ImportError, e: + else: try: from google.appengine.tools.devappserver2 import application_configuration configuration = application_configuration.ApplicationConfiguration(["app.yaml"]) From 64c4e640e155c5737dfe654a78c4a5e082558b5e Mon Sep 17 00:00:00 2001 From: projecteat Date: Mon, 25 Mar 2013 12:21:23 -0400 Subject: [PATCH 48/55] Revert changes to utils.py that were needed for the standalone devapppserver2. Make python manage.py deploy work. Removed the use of the appcfg -R flag, it shouldn't be necessary. --- djangoappengine/management/commands/deploy.py | 2 -- djangoappengine/utils.py | 16 ++++------------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/djangoappengine/management/commands/deploy.py b/djangoappengine/management/commands/deploy.py index dcc250e..303502f 100644 --- a/djangoappengine/management/commands/deploy.py +++ b/djangoappengine/management/commands/deploy.py @@ -29,8 +29,6 @@ def run_appcfg(argv): new_args = argv[:] new_args[1] = 'update' - if appconfig.runtime != 'python': - new_args.insert(1, '-R') new_args.append(PROJECT_DIR) syncdb = True if '--nosyncdb' in new_args: diff --git a/djangoappengine/utils.py b/djangoappengine/utils.py index 77f4f68..9cd0290 100644 --- a/djangoappengine/utils.py +++ b/djangoappengine/utils.py @@ -9,23 +9,15 @@ if have_appserver: appid = get_application_id() else: - from boot import devappserver_ver - if devappserver_ver == 1: - # Original dev_appserver method + try: from google.appengine.tools import dev_appserver from .boot import PROJECT_DIR appconfig = dev_appserver.LoadAppConfig(PROJECT_DIR, {}, default_partition='dev')[0] appid = appconfig.application.split('~', 1)[-1] - else: - try: - from google.appengine.tools.devappserver2 import application_configuration - configuration = application_configuration.ApplicationConfiguration(["app.yaml"]) - appid = configuration.app_id.split('~', 1)[-1] - - except Exception, e: - raise Exception("Could not get appid. Is your app.yaml file missing? " - "Error was: %s" % e) + except Exception, e: + raise Exception("Could not get appid. Is your app.yaml file missing? " + "Error was: %s" % e) on_production_server = have_appserver and \ not os.environ.get('SERVER_SOFTWARE', '').lower().startswith('devel') From 9f059e7edd482ad011d49fd088a69410d33a180d Mon Sep 17 00:00:00 2001 From: projecteat Date: Tue, 6 Aug 2013 21:43:33 -0400 Subject: [PATCH 49/55] Add __str__ and __unicode___ methods to BlobstoreFile so filenames are displayed without the blobkey (ie in form FileField) --- djangoappengine/storage.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/djangoappengine/storage.py b/djangoappengine/storage.py index e7fef82..0e8a5d3 100644 --- a/djangoappengine/storage.py +++ b/djangoappengine/storage.py @@ -127,6 +127,12 @@ def __init__(self, name, mode, storage): self._mode = mode self.blobstore_info = storage._get_blobinfo(name) + def __str__(self): + return self.blobstore_info.filename + + def __unicode__(self): + return self.blobstore_info.filename + @property def size(self): return self.blobstore_info.size From 7ed1b53dfeb23592011acacb28cf08ad4bddcb73 Mon Sep 17 00:00:00 2001 From: projecteat Date: Mon, 19 Aug 2013 17:10:33 -0400 Subject: [PATCH 50/55] Force dev_appserver datastore consistency when calling loaddata. This wasn't necessary on the original dev_appserver, but necessary on devappserver2 since it defaults to eventually consistent. We can remove the same call from testserver in this case. --- .../management/commands/loaddata.py | 18 ++++++++++++++++++ .../management/commands/testserver.py | 12 ------------ 2 files changed, 18 insertions(+), 12 deletions(-) create mode 100644 djangoappengine/management/commands/loaddata.py diff --git a/djangoappengine/management/commands/loaddata.py b/djangoappengine/management/commands/loaddata.py new file mode 100644 index 0000000..22d0b84 --- /dev/null +++ b/djangoappengine/management/commands/loaddata.py @@ -0,0 +1,18 @@ +from django.core.management.commands.loaddata import Command as OriginalCommand + +from google.appengine.api import apiproxy_stub_map +from google.appengine.datastore import datastore_stub_util + +class Command(OriginalCommand): + def handle(self, *fixture_labels, **options): + # Temporarily change consistency policy to force apply loaded data + datastore = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') + + orig_consistency_policy = datastore._consistency_policy + datastore.SetConsistencyPolicy(datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)) + + retval = super(Command, self).handle(*fixture_labels, **options) + + datastore.SetConsistencyPolicy(orig_consistency_policy) + + return retval diff --git a/djangoappengine/management/commands/testserver.py b/djangoappengine/management/commands/testserver.py index cc0a8b0..738435f 100644 --- a/djangoappengine/management/commands/testserver.py +++ b/djangoappengine/management/commands/testserver.py @@ -1,8 +1,5 @@ from django.core.management.base import BaseCommand -from google.appengine.api import apiproxy_stub_map -from google.appengine.datastore import datastore_stub_util - from optparse import make_option class Command(BaseCommand): @@ -47,18 +44,9 @@ def handle(self, *fixture_labels, **options): db_name = name break - # Temporarily change consistency policy to force apply loaded data - datastore = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') - - orig_consistency_policy = datastore._consistency_policy - datastore.SetConsistencyPolicy(datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)) - # Import the fixture data into the test database. call_command('loaddata', *fixture_labels, **{'verbosity': verbosity}) - # reset original policy - datastore.SetConsistencyPolicy(orig_consistency_policy) - # Run the development server. Turn off auto-reloading because it causes # a strange error -- it causes this handle() method to be called # multiple times. From 122b1fe2ff79e86511ace9c3c49afa48fde465c9 Mon Sep 17 00:00:00 2001 From: projecteat Date: Wed, 21 Aug 2013 02:12:56 -0400 Subject: [PATCH 51/55] Fix for last change not working properly on the original dev_appserver. Oops. --- djangoappengine/management/commands/loaddata.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/djangoappengine/management/commands/loaddata.py b/djangoappengine/management/commands/loaddata.py index 22d0b84..8b7285d 100644 --- a/djangoappengine/management/commands/loaddata.py +++ b/djangoappengine/management/commands/loaddata.py @@ -5,14 +5,21 @@ class Command(OriginalCommand): def handle(self, *fixture_labels, **options): + retval = None # Temporarily change consistency policy to force apply loaded data datastore = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') - orig_consistency_policy = datastore._consistency_policy - datastore.SetConsistencyPolicy(datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)) + try: + # datastore._consistency_policy only exists in dev_appserver + # will throw exception in production + orig_consistency_policy = datastore._consistency_policy + datastore.SetConsistencyPolicy(datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)) + except: + orig_consistency_policy = None retval = super(Command, self).handle(*fixture_labels, **options) - datastore.SetConsistencyPolicy(orig_consistency_policy) + if orig_consistency_policy: + datastore.SetConsistencyPolicy(orig_consistency_policy) return retval From ca5cd85ef99e4762c7c47a62659c57361a0be67c Mon Sep 17 00:00:00 2001 From: projecteat Date: Sat, 12 Oct 2013 01:34:01 -0400 Subject: [PATCH 52/55] Update input readers for reading keys and raw entities given Django models. --- djangoappengine/mapreduce/input_readers.py | 44 +++++++++++++++++++++- djangoappengine/mapreduce/inputreader.py | 31 --------------- 2 files changed, 42 insertions(+), 33 deletions(-) delete mode 100644 djangoappengine/mapreduce/inputreader.py diff --git a/djangoappengine/mapreduce/input_readers.py b/djangoappengine/mapreduce/input_readers.py index fb2740e..f6b9fbc 100644 --- a/djangoappengine/mapreduce/input_readers.py +++ b/djangoappengine/mapreduce/input_readers.py @@ -1,11 +1,51 @@ from djangoappengine.db.utils import get_cursor, set_cursor, set_config +from django.db.models.sql.query import Query -from google.appengine.api.datastore import Key +from google.appengine.datastore import datastore_query from mapreduce.datastore_range_iterators import AbstractKeyRangeIterator, _KEY_RANGE_ITERATORS -from mapreduce.input_readers import AbstractDatastoreInputReader, _get_params, BadReaderParamsError +from mapreduce.input_readers import AbstractDatastoreInputReader, RawDatastoreInputReader, _get_params, BadReaderParamsError from mapreduce import util +class DjangoKeyIterator(AbstractKeyRangeIterator): + """An iterator that takes a Django model ('app.models.Model') and yields Keys for that model""" + _KEYS_ONLY = True + def __iter__(self): + query = Query(util.for_name(self._query_spec.model_class_path)).get_compiler(using="default").build_query() + raw_entity_kind = query.db_table + + q = self._key_range.make_ascending_datastore_query(raw_entity_kind, keys_only=self._KEYS_ONLY) + if self._cursor: + q = set_cursor(q, self._cursor) + + self._query = q + for key in q.Run( + config=datastore_query.QueryOptions(batch_size=self._query_spec.batch_size)): + yield key + + def _get_cursor(self): + if self._query is not None: + return self._query.cursor() + +_KEY_RANGE_ITERATORS[DjangoKeyIterator.__name__] = DjangoKeyIterator + +class DjangoKeyInputReader(RawDatastoreInputReader): + """An input reader that takes a Django model ('app.models.Model') and yields keys for that model""" + _KEY_RANGE_ITER_CLS = DjangoKeyIterator + + +class DjangoRawEntityIterator(DjangoKeyIterator): + """An iterator that takes a Django model ('app.models.Model') and yields raw entities for that model""" + _KEYS_ONLY = False + +_KEY_RANGE_ITERATORS[DjangoRawEntityIterator.__name__] = DjangoRawEntityIterator + + +class DjangoRawEntityInputReader(RawDatastoreInputReader): + """An input reader that takes a Django model ('app.models.Model') and yields raw entities for that model""" + _KEY_RANGE_ITER_CLS = DjangoRawEntityIterator + + class DjangoModelIterator(AbstractKeyRangeIterator): def __iter__(self): k_range = self._key_range diff --git a/djangoappengine/mapreduce/inputreader.py b/djangoappengine/mapreduce/inputreader.py deleted file mode 100644 index aeae8d1..0000000 --- a/djangoappengine/mapreduce/inputreader.py +++ /dev/null @@ -1,31 +0,0 @@ -import djangoappengine.main - -from django.db.models.sql.query import Query - -from mapreduce.input_readers import AbstractDatastoreInputReader -from mapreduce import util -from google.appengine.datastore import datastore_query - -class DjangoKeyInputReader(AbstractDatastoreInputReader): - """An input reader that takes a Django model ('app.models.Model') and yields Keys for that model""" - - def _iter_key_range(self, k_range): - query = Query(util.for_name(self._entity_kind)).get_compiler(using="default").build_query() - raw_entity_kind = query.db_table - - query = k_range.make_ascending_datastore_query(raw_entity_kind, keys_only=True) - for key in query.Run( - config=datastore_query.QueryOptions(batch_size=self._batch_size)): - yield key, key - -class DjangoEntityInputReader(AbstractDatastoreInputReader): - """An input reader that takes a Django model ('app.models.Model') and yields entities for that model""" - - def _iter_key_range(self, k_range): - query = Query(util.for_name(self._entity_kind)).get_compiler(using="default").build_query() - raw_entity_kind = query.db_table - - query = k_range.make_ascending_datastore_query(raw_entity_kind) - for entity in query.Run( - config=datastore_query.QueryOptions(batch_size=self._batch_size)): - yield entity.key(), entity From a7a442645983b800740bb3e0eb04eecb1a6de7a5 Mon Sep 17 00:00:00 2001 From: projecteat Date: Wed, 30 Oct 2013 12:41:13 -0400 Subject: [PATCH 53/55] Patch tests so they don't throw an exception when calling the latest mapreduce library. --- djangoappengine/management/commands/test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/djangoappengine/management/commands/test.py b/djangoappengine/management/commands/test.py index c3372df..ae12d84 100644 --- a/djangoappengine/management/commands/test.py +++ b/djangoappengine/management/commands/test.py @@ -37,6 +37,15 @@ def my_encode_file(boundary, key, file): class Command(OriginalCommand): def __init__(self): + # Add a non-null version for DEFAULT_VERSION_HOSTNAME + # or else the mapreduce library's handler function + # throws an exception. This only prevents the exception. + import os + default_version_hostname = "mr-test-support.appspot.com" + if "DEFAULT_VERSION_HOSTNAME" not in os.environ: + os.environ["DEFAULT_VERSION_HOSTNAME"] = ( + default_version_hostname) + # monkey patch client's encode_file with our own # with blobstore support client.encode_file = my_encode_file From 85f5f150543ec7eff939d5bdab2ce8080ac8e42f Mon Sep 17 00:00:00 2001 From: projecteat Date: Wed, 30 Oct 2013 13:08:49 -0400 Subject: [PATCH 54/55] Attempt to fix django-nonrel Travis-CI test by disabling images stub. --- djangoappengine/db/stubs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/djangoappengine/db/stubs.py b/djangoappengine/db/stubs.py index 0f03016..1cc9adc 100644 --- a/djangoappengine/db/stubs.py +++ b/djangoappengine/db/stubs.py @@ -68,7 +68,7 @@ def activate_test_stubs(self, connection): self.testbed.init_channel_stub() self.testbed.init_files_stub(True) self.testbed.init_blobstore_stub(True) - self.testbed.init_images_stub(True) + self.testbed.init_images_stub(False) def deactivate_test_stubs(self): if self.active_stubs == 'test': @@ -86,7 +86,9 @@ def setup_local_stubs(self, connection): log_level = logging.getLogger().getEffectiveLevel() logging.getLogger().setLevel(logging.WARNING) from google.appengine.tools import dev_appserver - dev_appserver.SetupStubs('dev~' + appid, **args) + dev_appserver.SetupStubs('dev~' + appid, + _use_atexit_for_datastore_stub=True, + **args) logging.getLogger().setLevel(log_level) self.active_stubs = 'local' if devappserver_ver == 2: From 0376e4778079023bfb229da9fb7325f7b0ccf848 Mon Sep 17 00:00:00 2001 From: projecteat Date: Wed, 30 Oct 2013 17:04:50 -0400 Subject: [PATCH 55/55] Reinstating the init_images_stub, since it's required for the blobstore APIs. TODO: add PIL to Travis config --- djangoappengine/db/stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/djangoappengine/db/stubs.py b/djangoappengine/db/stubs.py index 1cc9adc..02583fc 100644 --- a/djangoappengine/db/stubs.py +++ b/djangoappengine/db/stubs.py @@ -68,7 +68,7 @@ def activate_test_stubs(self, connection): self.testbed.init_channel_stub() self.testbed.init_files_stub(True) self.testbed.init_blobstore_stub(True) - self.testbed.init_images_stub(False) + self.testbed.init_images_stub(True) def deactivate_test_stubs(self): if self.active_stubs == 'test':