From b3f9389644cbee50f1702df2c40c6e51940f87ca Mon Sep 17 00:00:00 2001 From: Alessio Fabiani Date: Mon, 4 Sep 2017 16:13:11 +0200 Subject: [PATCH] - Returns referencing infos on Layers Upload - Returns referencing infos on Layers Upload - Returns referencing infos on Layers Upload - Returns referencing infos on Layers Upload --- flake8.txt | 7 + geonode/api/resourcebase_api.py | 62 +-- geonode/base/forms.py | 150 ++++--- geonode/base/models.py | 550 ++++++++++++++++++-------- geonode/catalogue/backends/generic.py | 103 +++-- geonode/contrib/favorite/models.py | 13 +- geonode/contrib/mp/signals.py | 8 +- geonode/contrib/ows_api/urls.py | 2 +- geonode/documents/views.py | 84 ++-- geonode/layers/models.py | 47 ++- geonode/layers/utils.py | 66 ++-- geonode/layers/views.py | 7 + geonode/maps/models.py | 4 +- geonode/maps/views.py | 185 ++++++--- geonode/services/views.py | 338 ++++++++++------ geonode/upload/upload.py | 176 ++++++--- geonode/upload/views.py | 49 ++- geonode/utils.py | 76 ++-- pavement.py | 65 +-- setup.cfg | 3 +- 20 files changed, 1358 insertions(+), 637 deletions(-) create mode 100644 flake8.txt diff --git a/flake8.txt b/flake8.txt new file mode 100644 index 00000000000..3063e84500e --- /dev/null +++ b/flake8.txt @@ -0,0 +1,7 @@ +# Check formatting with + +flake8 --config=setup.cfg + +# Auto-fix formatting with + +autopep8 --global-config setup.cfg -i -a -a [files [files ...]] diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index cc1c3a524f8..cf4b3af2d73 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -86,7 +86,8 @@ class CommonModelApi(ModelResource): null=True, full=True) owner = fields.ToOneField(OwnersResource, 'owner', full=True) - tkeywords = fields.ToManyField(ThesaurusKeywordResource, 'tkeywords', null=True) + tkeywords = fields.ToManyField( + ThesaurusKeywordResource, 'tkeywords', null=True) VALUES = [ # fields in the db 'id', @@ -186,7 +187,7 @@ def filter_group(self, queryset, request): try: anonymous_group = Group.objects.get(name='anonymous') - except: + except BaseException: anonymous_group = None if is_admin: @@ -194,13 +195,15 @@ def filter_group(self, queryset, request): elif request.user: groups = request.user.groups.all() if anonymous_group: - filtered = queryset.filter( - Q(group__isnull=True) | Q(group__in=groups) | Q(group=anonymous_group)) + filtered = queryset.filter(Q(group__isnull=True) | Q( + group__in=groups) | Q(group=anonymous_group)) else: - filtered = queryset.filter(Q(group__isnull=True) | Q(group__in=groups)) + filtered = queryset.filter( + Q(group__isnull=True) | Q(group__in=groups)) else: if anonymous_group: - filtered = queryset.filter(Q(group__isnull=True) | Q(group=anonymous_group)) + filtered = queryset.filter( + Q(group__isnull=True) | Q(group=anonymous_group)) else: filtered = queryset.filter(Q(group__isnull=True)) return filtered @@ -432,7 +435,8 @@ def get_search(self, request, **kwargs): is_staff = request.user.is_staff if request.user else False # Get the list of objects the user has access to - filter_set = get_objects_for_user(request.user, 'base.view_resourcebase') + filter_set = get_objects_for_user( + request.user, 'base.view_resourcebase') if settings.ADMIN_MODERATE_UPLOADS: if not is_admin and not is_staff: filter_set = filter_set.filter(is_published=True) @@ -442,7 +446,7 @@ def get_search(self, request, **kwargs): try: anonymous_group = Group.objects.get(name='anonymous') - except: + except BaseException: anonymous_group = None if settings.GROUP_PRIVATE_RESOURCES: @@ -451,13 +455,15 @@ def get_search(self, request, **kwargs): elif request.user: groups = request.user.groups.all() if anonymous_group: - filter_set = filter_set.filter( - Q(group__isnull=True) | Q(group__in=groups) | Q(group=anonymous_group)) + filter_set = filter_set.filter(Q(group__isnull=True) | Q( + group__in=groups) | Q(group=anonymous_group)) else: - filter_set = filter_set.filter(Q(group__isnull=True) | Q(group__in=groups)) + filter_set = filter_set.filter( + Q(group__isnull=True) | Q(group__in=groups)) else: if anonymous_group: - filter_set = filter_set.filter(Q(group__isnull=True) | Q(group=anonymous_group)) + filter_set = filter_set.filter( + Q(group__isnull=True) | Q(group=anonymous_group)) else: filter_set = filter_set.filter(Q(group__isnull=True)) @@ -465,8 +471,8 @@ def get_search(self, request, **kwargs): # Do the query using the filterset and the query term. Facet the # results if len(filter_set) > 0: - sqs = sqs.filter(id__in=filter_set_ids).facet('type').facet('subtype').facet('owner')\ - .facet('keywords').facet('regions').facet('category') + sqs = sqs.filter(id__in=filter_set_ids).facet('type').facet('subtype').facet( + 'owner') .facet('keywords').facet('regions').facet('category') else: sqs = None else: @@ -509,7 +515,7 @@ def get_search(self, request, **kwargs): objects = [] object_list = { - "meta": { + "meta": { "limit": settings.API_LIMIT_PER_PAGE, "next": next_page, "offset": int(getattr(request.GET, 'offset', 0)), @@ -517,15 +523,16 @@ def get_search(self, request, **kwargs): "total_count": total_count, "facets": facets, }, - "objects": map(lambda x: self.get_haystack_api_fields(x), objects), + "objects": map(lambda x: self.get_haystack_api_fields(x), objects), } self.log_throttled_access(request) return self.create_response(request, object_list) def get_haystack_api_fields(self, haystack_object): - object_fields = dict((k, v) for k, v in haystack_object.get_stored_fields().items() - if not re.search('_exact$|_sortable$', k)) + object_fields = dict( + (k, v) for k, v in haystack_object.get_stored_fields().items() if not re.search( + '_exact$|_sortable$', k)) return object_fields def get_list(self, request, **kwargs): @@ -558,7 +565,8 @@ def get_list(self, request, **kwargs): request, to_be_serialized) - return self.create_response(request, to_be_serialized, response_objects=objects) + return self.create_response( + request, to_be_serialized, response_objects=objects) def format_objects(self, objects): """ @@ -579,11 +587,13 @@ def create_response( Mostly a useful shortcut/hook. """ - # If an user does not have at least view permissions, he won't be able to see the resource at all. + # If an user does not have at least view permissions, he won't be able + # to see the resource at all. filtered_objects_ids = None if response_objects: - filtered_objects_ids = [item.id for item in response_objects if - request.user.has_perm('view_resourcebase', item.get_self_resource())] + filtered_objects_ids = [ + item.id for item in response_objects if request.user.has_perm( + 'view_resourcebase', item.get_self_resource())] if isinstance( data, @@ -591,8 +601,10 @@ def create_response( data['objects'], list): if filtered_objects_ids: - data['objects'] = [x for x in list(self.format_objects(data['objects'])) - if x['id'] in filtered_objects_ids] + data['objects'] = [ + x for x in list( + self.format_objects( + data['objects'])) if x['id'] in filtered_objects_ids] else: data['objects'] = list(self.format_objects(data['objects'])) @@ -609,7 +621,7 @@ def prepend_urls(self): return [ url(r"^(?P%s)/search%s$" % ( self._meta.resource_name, trailing_slash() - ), + ), self.wrap_view('get_search'), name="api_get_search"), ] else: diff --git a/geonode/base/forms.py b/geonode/base/forms.py index fe4613b5344..7e871b366b2 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -75,7 +75,7 @@ def rectree(parent, path): data.append( tuple((toplevel.name, childrens)) ) - except: + except BaseException: pass return tuple(data) @@ -83,7 +83,10 @@ def rectree(parent, path): class AdvancedModelChoiceIterator(models.ModelChoiceIterator): def choice(self, obj): - return (self.field.prepare_value(obj), self.field.label_from_instance(obj), obj) + return ( + self.field.prepare_value(obj), + self.field.label_from_instance(obj), + obj) class CategoryChoiceField(forms.ModelChoiceField): @@ -96,33 +99,35 @@ def _get_choices(self): choices = property(_get_choices, ChoiceField._set_choices) def label_from_instance(self, obj): - return '' \ - '' \ + return '' \ + '' \ '' \ '
' + obj.gn_description + '' class TreeWidget(TaggitWidget): - input_type = 'text' - - def render(self, name, values, attrs=None): - if isinstance(values, basestring): - vals = values - elif values: - vals = ','.join([str(i.tag.name) for i in values]) - else: - vals = "" - output = ["""
+ input_type = 'text' + + def render(self, name, values, attrs=None): + if isinstance(values, basestring): + vals = values + elif values: + vals = ','.join([str(i.tag.name) for i in values]) + else: + vals = "" + output = ["""

""" % (vals)] - output.append('') - output.append('') - output.append('
') + output.append( + '') + output.append( + '') + output.append('
') - return mark_safe(u'\n'.join(output)) + return mark_safe(u'\n'.join(output)) class RegionsMultipleChoiceField(forms.MultipleChoiceField): @@ -131,7 +136,8 @@ def validate(self, value): Validates that the input is a list or tuple. """ if self.required and not value: - raise forms.ValidationError(self.error_messages['required'], code='required') + raise forms.ValidationError( + self.error_messages['required'], code='required') class RegionsSelect(forms.Select): @@ -141,7 +147,10 @@ def render(self, name, value, attrs=None): if value is None: value = [] final_attrs = self.build_attrs(attrs, name=name) - output = [format_html('', + flatatt(final_attrs))] options = self.render_options(value) if options: output.append(options) @@ -155,7 +164,12 @@ def value_from_datadict(self, data, files, name): getter = data.get return getter(name) - def render_option_value(self, selected_choices, option_value, option_label, data_section=None): + def render_option_value( + self, + selected_choices, + option_value, + option_label, + data_section=None): if option_value is None: option_value = '' option_value = force_text(option_value) @@ -174,13 +188,16 @@ def render_option_value(self, selected_choices, option_value, option_label, data else: data_section = force_text(data_section) if '/' in data_section: - label = format_html('{} [{}]', label, data_section.rsplit('/', 1)[1]) + label = format_html( + '{} [{}]', label, data_section.rsplit( + '/', 1)[1]) - return format_html('', - data_section, - option_value, - selected_html, - label) + return format_html( + '', + data_section, + option_value, + selected_html, + label) def render_options(self, selected_choices): # Normalize to strings. @@ -189,39 +206,64 @@ def render_options(self, selected_choices): output.append(format_html('', 'Global')) for option_value, option_label in self.choices: - if not isinstance(option_label, (list, tuple)) and isinstance(option_label, basestring): - output.append(self.render_option_value(selected_choices, option_value, option_label)) + if not isinstance( + option_label, (list, tuple)) and isinstance( + option_label, basestring): + output.append( + self.render_option_value( + selected_choices, + option_value, + option_label)) output.append('') for option_value, option_label in self.choices: - if isinstance(option_label, (list, tuple)) and not isinstance(option_label, basestring): - output.append(format_html('', force_text(option_value))) + if isinstance( + option_label, (list, tuple)) and not isinstance( + option_label, basestring): + output.append( + format_html( + '', + force_text(option_value))) for option in option_label: - if isinstance(option, (list, tuple)) and not isinstance(option, basestring): - if isinstance(option[1][0], (list, tuple)) and not isinstance(option[1][0], basestring): + if isinstance( + option, (list, tuple)) and not isinstance( + option, basestring): + if isinstance( + option[1][0], (list, tuple)) and not isinstance( + option[1][0], basestring): for option_child in option[1][0]: - output.append(self.render_option_value(selected_choices, - *option_child, - data_section=force_text(option[1][0][0]))) + output.append( + self.render_option_value( + selected_choices, + *option_child, + data_section=force_text( + option[1][0][0]))) else: - output.append(self.render_option_value(selected_choices, - *option[1], - data_section=force_text(option[0]))) + output.append( + self.render_option_value( + selected_choices, + *option[1], + data_section=force_text( + option[0]))) else: - output.append(self.render_option_value(selected_choices, - *option, - data_section=force_text(option_value))) + output.append( + self.render_option_value( + selected_choices, + *option, + data_section=force_text(option_value))) output.append('') return '\n'.join(output) class CategoryForm(forms.Form): - category_choice_field = CategoryChoiceField(required=False, - label='*' + _('Category'), - empty_label=None, - queryset=TopicCategory.objects.filter(is_choice=True) - .extra(order_by=['description'])) + category_choice_field = CategoryChoiceField( + required=False, + label='*' + _('Category'), + empty_label=None, + queryset=TopicCategory.objects.filter( + is_choice=True) .extra( + order_by=['description'])) def clean(self): cleaned_data = self.data @@ -246,10 +288,9 @@ def clean(self): cleaned_data = None if self.data: try: - cleaned_data = [{key: self.data.getlist(key)} for key, value - in self.data.items() - if 'tkeywords-tkeywords' in key.lower() and 'autocomplete' not in key.lower()] - except: + cleaned_data = [{key: self.data.getlist(key)} for key, value in self.data.items( + ) if 'tkeywords-tkeywords' in key.lower() and 'autocomplete' not in key.lower()] + except BaseException: pass return cleaned_data @@ -277,7 +318,7 @@ class ResourceBaseForm(TranslationModelForm): # and we need to ensure it is available before trying to # instantiate a new datetimepicker. This could probably be improved. "options": False, - } + } date = forms.DateTimeField( label=_("Date"), localize=True, @@ -319,7 +360,8 @@ class ResourceBaseForm(TranslationModelForm): label=_("Free-text Keywords"), required=False, help_text=_("A space or comma-separated list of keywords. Use the widget to select from Hierarchical tree."), - widget=TreeWidget(autocomplete='HierarchicalKeywordAutocomplete')) + widget=TreeWidget( + autocomplete='HierarchicalKeywordAutocomplete')) """ regions = TreeNodeMultipleChoiceField( @@ -374,7 +416,7 @@ class Meta: 'charset', 'rating', 'detail_url' - ) + ) class ValuesListField(forms.Field): diff --git a/geonode/base/models.py b/geonode/base/models.py index 6abc10c23d2..a7df8182431 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -77,29 +77,43 @@ class ContactRole(models.Model): """ resource = models.ForeignKey('ResourceBase', blank=True, null=True) contact = models.ForeignKey(settings.AUTH_USER_MODEL) - role = models.CharField(choices=ROLE_VALUES, max_length=255, help_text=_('function performed by the responsible ' - 'party')) + role = models.CharField( + choices=ROLE_VALUES, + max_length=255, + help_text=_( + 'function performed by the responsible ' + 'party')) def clean(self): """ Make sure there is only one poc and author per resource """ - if (self.role == self.resource.poc_role) or (self.role == self.resource.metadata_author_role): - contacts = self.resource.contacts.filter(contactrole__role=self.role) + if (self.role == self.resource.poc_role) or ( + self.role == self.resource.metadata_author_role): + contacts = self.resource.contacts.filter( + contactrole__role=self.role) if contacts.count() == 1: # only allow this if we are updating the same contact if self.contact != contacts.get(): - raise ValidationError('There can be only one %s for a given resource' % self.role) + raise ValidationError( + 'There can be only one %s for a given resource' % + self.role) if self.contact.user is None: - # verify that any unbound contact is only associated to one resource + # verify that any unbound contact is only associated to one + # resource bounds = ContactRole.objects.filter(contact=self.contact).count() if bounds > 1: - raise ValidationError('There can be one and only one resource linked to an unbound contact' % self.role) + raise ValidationError( + 'There can be one and only one resource linked to an unbound contact' % + self.role) elif bounds == 1: - # verify that if there was one already, it corresponds to this instance - if ContactRole.objects.filter(contact=self.contact).get().id != self.id: - raise ValidationError('There can be one and only one resource linked to an unbound contact' - % self.role) + # verify that if there was one already, it corresponds to this + # instance + if ContactRole.objects.filter( + contact=self.contact).get().id != self.id: + raise ValidationError( + 'There can be one and only one resource linked to an unbound contact' % + self.role) class Meta: unique_together = (("contact", "resource", "role"),) @@ -114,7 +128,8 @@ class TopicCategory(models.Model): """ identifier = models.CharField(max_length=255, default='location') description = models.TextField(default='') - gn_description = models.TextField('GeoNode description', default='', null=True) + gn_description = models.TextField( + 'GeoNode description', default='', null=True) is_choice = models.BooleanField(default=True) fa_class = models.CharField(max_length=64, default='fa-times') @@ -156,14 +171,35 @@ class Region(MPTTModel): code = models.CharField(max_length=50, unique=True) name = models.CharField(max_length=255) - parent = TreeForeignKey('self', null=True, blank=True, related_name='children') + parent = TreeForeignKey( + 'self', + null=True, + blank=True, + related_name='children') # Save bbox values in the database. - # This is useful for spatial searches and for generating thumbnail images and metadata records. - bbox_x0 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) - bbox_x1 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) - bbox_y0 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) - bbox_y1 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) + # This is useful for spatial searches and for generating thumbnail images + # and metadata records. + bbox_x0 = models.DecimalField( + max_digits=19, + decimal_places=10, + blank=True, + null=True) + bbox_x1 = models.DecimalField( + max_digits=19, + decimal_places=10, + blank=True, + null=True) + bbox_y0 = models.DecimalField( + max_digits=19, + decimal_places=10, + blank=True, + null=True) + bbox_y1 = models.DecimalField( + max_digits=19, + decimal_places=10, + blank=True, + null=True) srid = models.CharField(max_length=255, default='EPSG:4326') def __unicode__(self): @@ -171,15 +207,26 @@ def __unicode__(self): @property def bbox(self): - return [self.bbox_x0, self.bbox_x1, self.bbox_y0, self.bbox_y1, self.srid] + return [ + self.bbox_x0, + self.bbox_x1, + self.bbox_y0, + self.bbox_y1, + self.srid] @property def bbox_string(self): - return ",".join([str(self.bbox_x0), str(self.bbox_y0), str(self.bbox_x1), str(self.bbox_y1)]) + return ",".join([str(self.bbox_x0), str(self.bbox_y0), + str(self.bbox_x1), str(self.bbox_y1)]) @property def geographic_bounding_box(self): - return bbox_to_wkt(self.bbox_x0, self.bbox_x1, self.bbox_y0, self.bbox_y1, srid=self.srid) + return bbox_to_wkt( + self.bbox_x0, + self.bbox_x1, + self.bbox_y0, + self.bbox_y1, + srid=self.srid) class Meta: ordering = ("name",) @@ -238,7 +285,7 @@ def name_long(self): if self.abbreviation is None or len(self.abbreviation) == 0: return self.name else: - return self.name+" ("+self.abbreviation+")" + return self.name + " (" + self.abbreviation + ")" @property def description_bullets(self): @@ -248,7 +295,7 @@ def description_bullets(self): bullets = [] lines = self.description.split("\n") for line in lines: - bullets.append("+ "+line) + bullets.append("+ " + line) return bullets class Meta: @@ -335,14 +382,19 @@ def add(self, *tags): tag_objs.add(HierarchicalKeyword.add_root(name=new_tag)) for tag in tag_objs: - self.through.objects.get_or_create(tag=tag, **self._lookup_kwargs()) + self.through.objects.get_or_create( + tag=tag, **self._lookup_kwargs()) class Thesaurus(models.Model): """ Loadable thesaurus containing keywords in different languages """ - identifier = models.CharField(max_length=255, null=False, blank=False, unique=True) + identifier = models.CharField( + max_length=255, + null=False, + blank=False, + unique=True) # read from the RDF file title = models.CharField(max_length=255, null=False, blank=False) @@ -368,7 +420,11 @@ class ThesaurusKeyword(models.Model): # read from the RDF file about = models.CharField(max_length=255, null=True, blank=True) # read from the RDF file - alt_label = models.CharField(max_length=255, default='', null=True, blank=True) + alt_label = models.CharField( + max_length=255, + default='', + null=True, + blank=True) thesaurus = models.ForeignKey('Thesaurus', related_name='thesaurus') @@ -408,12 +464,15 @@ def admin_contact(self): # this assumes there is at least one superuser superusers = get_user_model().objects.filter(is_superuser=True).order_by('id') if superusers.count() == 0: - raise RuntimeError('GeoNode needs at least one admin/superuser set') + raise RuntimeError( + 'GeoNode needs at least one admin/superuser set') return superusers[0] def get_queryset(self): - return super(ResourceBaseManager, self).get_queryset().non_polymorphic() + return super( + ResourceBaseManager, + self).get_queryset().non_polymorphic() def polymorphic_queryset(self): return super(ResourceBaseManager, self).get_queryset() @@ -424,88 +483,177 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): Base Resource Object loosely based on ISO 19115:2003 """ - VALID_DATE_TYPES = [(x.lower(), _(x)) for x in ['Creation', 'Publication', 'Revision']] + VALID_DATE_TYPES = [(x.lower(), _(x)) + for x in ['Creation', 'Publication', 'Revision']] date_help_text = _('reference date for the cited resource') date_type_help_text = _('identification of when a given event occurred') edition_help_text = _('version of the cited resource') - abstract_help_text = _('brief narrative summary of the content of the resource(s)') - purpose_help_text = _('summary of the intentions with which the resource(s) was developed') - maintenance_frequency_help_text = _('frequency with which modifications and deletions are made to the data after ' - 'it is first produced') - keywords_help_text = _('commonly used word(s) or formalised word(s) or phrase(s) used to describe the subject ' - '(space or comma-separated') - tkeywords_help_text = _('formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject ' - '(space or comma-separated') + abstract_help_text = _( + 'brief narrative summary of the content of the resource(s)') + purpose_help_text = _( + 'summary of the intentions with which the resource(s) was developed') + maintenance_frequency_help_text = _( + 'frequency with which modifications and deletions are made to the data after ' + 'it is first produced') + keywords_help_text = _( + 'commonly used word(s) or formalised word(s) or phrase(s) used to describe the subject ' + '(space or comma-separated') + tkeywords_help_text = _( + 'formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject ' + '(space or comma-separated') regions_help_text = _('keyword identifies a location') - restriction_code_type_help_text = _('limitation(s) placed upon the access or use of the data.') - constraints_other_help_text = _('other restrictions and legal prerequisites for accessing and using the resource or' - ' metadata') + restriction_code_type_help_text = _( + 'limitation(s) placed upon the access or use of the data.') + constraints_other_help_text = _( + 'other restrictions and legal prerequisites for accessing and using the resource or' + ' metadata') license_help_text = _('license of the dataset') language_help_text = _('language used within the dataset') - category_help_text = _('high-level geographic data thematic classification to assist in the grouping and search of ' - 'available geographic data sets.') - spatial_representation_type_help_text = _('method used to represent geographic information in the dataset.') - temporal_extent_start_help_text = _('time period covered by the content of the dataset (start)') - temporal_extent_end_help_text = _('time period covered by the content of the dataset (end)') - data_quality_statement_help_text = _('general explanation of the data producer\'s knowledge about the lineage of a' - ' dataset') + category_help_text = _( + 'high-level geographic data thematic classification to assist in the grouping and search of ' + 'available geographic data sets.') + spatial_representation_type_help_text = _( + 'method used to represent geographic information in the dataset.') + temporal_extent_start_help_text = _( + 'time period covered by the content of the dataset (start)') + temporal_extent_end_help_text = _( + 'time period covered by the content of the dataset (end)') + data_quality_statement_help_text = _( + 'general explanation of the data producer\'s knowledge about the lineage of a' + ' dataset') # internal fields uuid = models.CharField(max_length=36) - owner = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True, related_name='owned_resource', - verbose_name=_("Owner")) - contacts = models.ManyToManyField(settings.AUTH_USER_MODEL, through='ContactRole') - title = models.CharField(_('title'), max_length=255, help_text=_('name by which the cited resource is known')) + owner = models.ForeignKey( + settings.AUTH_USER_MODEL, + blank=True, + null=True, + related_name='owned_resource', + verbose_name=_("Owner")) + contacts = models.ManyToManyField( + settings.AUTH_USER_MODEL, + through='ContactRole') + title = models.CharField(_('title'), max_length=255, help_text=_( + 'name by which the cited resource is known')) alternate = models.CharField(max_length=128, null=True, blank=True) - date = models.DateTimeField(_('date'), default=datetime.datetime.now, help_text=date_help_text) - date_type = models.CharField(_('date type'), max_length=255, choices=VALID_DATE_TYPES, default='publication', - help_text=date_type_help_text) - edition = models.CharField(_('edition'), max_length=255, blank=True, null=True, help_text=edition_help_text) - abstract = models.TextField(_('abstract'), max_length=2000, blank=True, help_text=abstract_help_text) - purpose = models.TextField(_('purpose'), max_length=500, null=True, blank=True, help_text=purpose_help_text) - maintenance_frequency = models.CharField(_('maintenance frequency'), max_length=255, choices=UPDATE_FREQUENCIES, - blank=True, null=True, help_text=maintenance_frequency_help_text) - - keywords = TaggableManager(_('keywords'), through=TaggedContentItem, blank=True, help_text=keywords_help_text, - manager=_HierarchicalTagManager) - tkeywords = models.ManyToManyField(ThesaurusKeyword, help_text=tkeywords_help_text, blank=True) - regions = models.ManyToManyField(Region, verbose_name=_('keywords region'), blank=True, - help_text=regions_help_text) - - restriction_code_type = models.ForeignKey(RestrictionCodeType, verbose_name=_('restrictions'), - help_text=restriction_code_type_help_text, null=True, blank=True, - limit_choices_to=Q(is_choice=True)) - - constraints_other = models.TextField(_('restrictions other'), blank=True, null=True, - help_text=constraints_other_help_text) + date = models.DateTimeField( + _('date'), + default=datetime.datetime.now, + help_text=date_help_text) + date_type = models.CharField( + _('date type'), + max_length=255, + choices=VALID_DATE_TYPES, + default='publication', + help_text=date_type_help_text) + edition = models.CharField( + _('edition'), + max_length=255, + blank=True, + null=True, + help_text=edition_help_text) + abstract = models.TextField( + _('abstract'), + max_length=2000, + blank=True, + help_text=abstract_help_text) + purpose = models.TextField( + _('purpose'), + max_length=500, + null=True, + blank=True, + help_text=purpose_help_text) + maintenance_frequency = models.CharField( + _('maintenance frequency'), + max_length=255, + choices=UPDATE_FREQUENCIES, + blank=True, + null=True, + help_text=maintenance_frequency_help_text) + + keywords = TaggableManager( + _('keywords'), + through=TaggedContentItem, + blank=True, + help_text=keywords_help_text, + manager=_HierarchicalTagManager) + tkeywords = models.ManyToManyField( + ThesaurusKeyword, + help_text=tkeywords_help_text, + blank=True) + regions = models.ManyToManyField( + Region, + verbose_name=_('keywords region'), + blank=True, + help_text=regions_help_text) + + restriction_code_type = models.ForeignKey( + RestrictionCodeType, + verbose_name=_('restrictions'), + help_text=restriction_code_type_help_text, + null=True, + blank=True, + limit_choices_to=Q( + is_choice=True)) + + constraints_other = models.TextField( + _('restrictions other'), + blank=True, + null=True, + help_text=constraints_other_help_text) license = models.ForeignKey(License, null=True, blank=True, verbose_name=_("License"), help_text=license_help_text) - language = models.CharField(_('language'), max_length=3, choices=ALL_LANGUAGES, default='eng', - help_text=language_help_text) - - category = models.ForeignKey(TopicCategory, null=True, blank=True, limit_choices_to=Q(is_choice=True), - help_text=category_help_text) - - spatial_representation_type = models.ForeignKey(SpatialRepresentationType, null=True, blank=True, - limit_choices_to=Q(is_choice=True), - verbose_name=_("spatial representation type"), - help_text=spatial_representation_type_help_text) + language = models.CharField( + _('language'), + max_length=3, + choices=ALL_LANGUAGES, + default='eng', + help_text=language_help_text) + + category = models.ForeignKey( + TopicCategory, + null=True, + blank=True, + limit_choices_to=Q( + is_choice=True), + help_text=category_help_text) + + spatial_representation_type = models.ForeignKey( + SpatialRepresentationType, + null=True, + blank=True, + limit_choices_to=Q( + is_choice=True), + verbose_name=_("spatial representation type"), + help_text=spatial_representation_type_help_text) # Section 5 - temporal_extent_start = models.DateTimeField(_('temporal extent start'), blank=True, null=True, - help_text=temporal_extent_start_help_text) - temporal_extent_end = models.DateTimeField(_('temporal extent end'), blank=True, null=True, - help_text=temporal_extent_end_help_text) - - supplemental_information = models.TextField(_('supplemental information'), max_length=2000, - default=DEFAULT_SUPPLEMENTAL_INFORMATION, - help_text=_('any other descriptive information about the dataset')) + temporal_extent_start = models.DateTimeField( + _('temporal extent start'), + blank=True, + null=True, + help_text=temporal_extent_start_help_text) + temporal_extent_end = models.DateTimeField( + _('temporal extent end'), + blank=True, + null=True, + help_text=temporal_extent_end_help_text) + + supplemental_information = models.TextField( + _('supplemental information'), + max_length=2000, + default=DEFAULT_SUPPLEMENTAL_INFORMATION, + help_text=_('any other descriptive information about the dataset')) # Section 8 - data_quality_statement = models.TextField(_('data quality statement'), max_length=2000, blank=True, null=True, - help_text=data_quality_statement_help_text) + data_quality_statement = models.TextField( + _('data quality statement'), + max_length=2000, + blank=True, + null=True, + help_text=data_quality_statement_help_text) group = models.ForeignKey(Group, null=True, blank=True) @@ -513,43 +661,78 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): # see metadata_author property definition below # Save bbox values in the database. - # This is useful for spatial searches and for generating thumbnail images and metadata records. - bbox_x0 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) - bbox_x1 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) - bbox_y0 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) - bbox_y1 = models.DecimalField(max_digits=19, decimal_places=10, blank=True, null=True) + # This is useful for spatial searches and for generating thumbnail images + # and metadata records. + bbox_x0 = models.DecimalField( + max_digits=19, + decimal_places=10, + blank=True, + null=True) + bbox_x1 = models.DecimalField( + max_digits=19, + decimal_places=10, + blank=True, + null=True) + bbox_y0 = models.DecimalField( + max_digits=19, + decimal_places=10, + blank=True, + null=True) + bbox_y1 = models.DecimalField( + max_digits=19, + decimal_places=10, + blank=True, + null=True) srid = models.CharField(max_length=255, default='EPSG:4326') # CSW specific fields - csw_typename = models.CharField(_('CSW typename'), max_length=32, default='gmd:MD_Metadata', null=False) + csw_typename = models.CharField( + _('CSW typename'), + max_length=32, + default='gmd:MD_Metadata', + null=False) csw_schema = models.CharField(_('CSW schema'), max_length=64, default='http://www.isotc211.org/2005/gmd', null=False) - csw_mdsource = models.CharField(_('CSW source'), max_length=256, default='local', null=False) - csw_insert_date = models.DateTimeField(_('CSW insert date'), auto_now_add=True, null=True) - csw_type = models.CharField(_('CSW type'), max_length=32, default='dataset', null=False, choices=HIERARCHY_LEVELS) + csw_mdsource = models.CharField( + _('CSW source'), + max_length=256, + default='local', + null=False) + csw_insert_date = models.DateTimeField( + _('CSW insert date'), auto_now_add=True, null=True) + csw_type = models.CharField( + _('CSW type'), + max_length=32, + default='dataset', + null=False, + choices=HIERARCHY_LEVELS) csw_anytext = models.TextField(_('CSW anytext'), null=True, blank=True) - csw_wkt_geometry = models.TextField(_('CSW WKT geometry'), - null=False, - default='POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))') + csw_wkt_geometry = models.TextField( + _('CSW WKT geometry'), + null=False, + default='POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))') # metadata XML specific fields metadata_uploaded = models.BooleanField(default=False) metadata_uploaded_preserve = models.BooleanField(default=False) - metadata_xml = models.TextField(null=True, - default='', - blank=True) + metadata_xml = models.TextField( + null=True, + default='', + blank=True) popular_count = models.IntegerField(default=0) share_count = models.IntegerField(default=0) - featured = models.BooleanField(_("Featured"), default=False, - help_text=_('Should this resource be advertised in home page?')) - is_published = models.BooleanField(_("Is Published"), default=True, - help_text=_('Should this resource be published and searchable?')) + featured = models.BooleanField(_("Featured"), default=False, help_text=_( + 'Should this resource be advertised in home page?')) + is_published = models.BooleanField( + _("Is Published"), + default=True, + help_text=_('Should this resource be published and searchable?')) # fields necessary for the apis thumbnail_url = models.TextField(null=True, blank=True) @@ -567,15 +750,26 @@ def group_name(self): @property def bbox(self): - return [self.bbox_x0, self.bbox_x1, self.bbox_y0, self.bbox_y1, self.srid] + return [ + self.bbox_x0, + self.bbox_x1, + self.bbox_y0, + self.bbox_y1, + self.srid] @property def bbox_string(self): - return ",".join([str(self.bbox_x0), str(self.bbox_y0), str(self.bbox_x1), str(self.bbox_y1)]) + return ",".join([str(self.bbox_x0), str(self.bbox_y0), + str(self.bbox_x1), str(self.bbox_y1)]) @property def geographic_bounding_box(self): - return bbox_to_wkt(self.bbox_x0, self.bbox_x1, self.bbox_y0, self.bbox_y1, srid=self.srid) + return bbox_to_wkt( + self.bbox_x0, + self.bbox_x1, + self.bbox_y0, + self.bbox_y1, + srid=self.srid) @property def license_light(self): @@ -585,18 +779,20 @@ def license_light(self): if (not (self.license.name is None)) and (len(self.license.name) > 0): a.append(self.license.name) if (not (self.license.url is None)) and (len(self.license.url) > 0): - a.append("("+self.license.url+")") + a.append("(" + self.license.url + ")") return " ".join(a) @property def license_verbose(self): a = [] - if (not (self.license.name_long is None)) and (len(self.license.name_long) > 0): - a.append(self.license.name_long+":") - if (not (self.license.description is None)) and (len(self.license.description) > 0): + if (not (self.license.name_long is None)) and ( + len(self.license.name_long) > 0): + a.append(self.license.name_long + ":") + if (not (self.license.description is None)) and ( + len(self.license.description) > 0): a.append(self.license.description) if (not (self.license.url is None)) and (len(self.license.url) > 0): - a.append("("+self.license.url+")") + a.append("(" + self.license.url + ")") return " ".join(a) @property @@ -692,7 +888,7 @@ def deg_len(): # taken from http://wiki.openstreetmap.org/wiki/Zoom_levels # it might be not precise but enough for the purpose - distance_per_pixel = 40075160 * math.cos(lat)/2**(zoom+8) + distance_per_pixel = 40075160 * math.cos(lat) / 2**(zoom + 8) # calculate the distance from the center of the map in degrees # we use the calculated degree length on the x axis and the @@ -739,12 +935,20 @@ def download_links(self): if url.link_type == 'metadata': # avoid recursion continue if url.link_type == 'html': - links.append((self.title, 'Web address (URL)', 'WWW:LINK-1.0-http--link', url.url)) + links.append( + (self.title, + 'Web address (URL)', + 'WWW:LINK-1.0-http--link', + url.url)) elif url.link_type in ('OGC:WMS', 'OGC:WFS', 'OGC:WCS'): links.append((self.title, url.name, url.link_type, url.url)) else: description = '%s (%s Format)' % (self.title, url.name) - links.append((self.title, description, 'WWW:DOWNLOAD-1.0-http--download', url.url)) + links.append( + (self.title, + description, + 'WWW:DOWNLOAD-1.0-http--download', + url.url)) return links def get_tiles_url(self): @@ -825,7 +1029,12 @@ def save_thumbnail(self, filename, image): storage.save(upload_path, ContentFile(image)) - url_path = os.path.join(settings.MEDIA_URL, upload_to, filename).replace('\\', '/') + url_path = os.path.join( + settings.MEDIA_URL, + upload_to, + filename).replace( + '\\', + '/') url = urljoin(settings.SITEURL, url_path) Link.objects.get_or_create(resource=self, @@ -842,7 +1051,9 @@ def save_thumbnail(self, filename, image): ) except Exception: - logger.error('Error when generating the thumbnail for resource %s.' % self.id) + logger.error( + 'Error when generating the thumbnail for resource %s.' % + self.id) logger.error('Check permissions for file %s.' % upload_path) def set_missing_info(self): @@ -855,12 +1066,13 @@ def set_missing_info(self): logger.debug('Checking for permissions.') # True if every key in the get_all_level_info dict is empty. no_custom_permissions = UserObjectPermission.objects.filter( - content_type=ContentType.objects.get_for_model(self.get_self_resource()), - object_pk=str(self.pk) - ).exists() + content_type=ContentType.objects.get_for_model( + self.get_self_resource()), object_pk=str( + self.pk)).exists() if not no_custom_permissions: - logger.debug('There are no permissions for this object, setting default perms.') + logger.debug( + 'There are no permissions for this object, setting default perms.') self.set_default_permissions() user = None @@ -869,7 +1081,7 @@ def set_missing_info(self): else: try: user = ResourceBase.objects.admin_contact().user - except: + except BaseException: pass if user: @@ -879,20 +1091,28 @@ def set_missing_info(self): self.metadata_author = user def maintenance_frequency_title(self): - return [v for i, v in enumerate(UPDATE_FREQUENCIES) if v[0] == self.maintenance_frequency][0][1].title() + return [v for i, v in enumerate( + UPDATE_FREQUENCIES) if v[0] == self.maintenance_frequency][0][1].title() def language_title(self): - return [v for i, v in enumerate(ALL_LANGUAGES) if v[0] == self.language][0][1].title() + return [v for i, v in enumerate( + ALL_LANGUAGES) if v[0] == self.language][0][1].title() def _set_poc(self, poc): # reset any poc assignation to this resource - ContactRole.objects.filter(role='pointOfContact', resource=self).delete() + ContactRole.objects.filter( + role='pointOfContact', + resource=self).delete() # create the new assignation - ContactRole.objects.create(role='pointOfContact', resource=self, contact=poc) + ContactRole.objects.create( + role='pointOfContact', + resource=self, + contact=poc) def _get_poc(self): try: - the_poc = ContactRole.objects.get(role='pointOfContact', resource=self).contact + the_poc = ContactRole.objects.get( + role='pointOfContact', resource=self).contact except ContactRole.DoesNotExist: the_poc = None return the_poc @@ -903,11 +1123,15 @@ def _set_metadata_author(self, metadata_author): # reset any metadata_author assignation to this resource ContactRole.objects.filter(role='author', resource=self).delete() # create the new assignation - ContactRole.objects.create(role='author', resource=self, contact=metadata_author) + ContactRole.objects.create( + role='author', + resource=self, + contact=metadata_author) def _get_metadata_author(self): try: - the_ma = ContactRole.objects.get(role='author', resource=self).contact + the_ma = ContactRole.objects.get( + role='author', resource=self).contact except ContactRole.DoesNotExist: the_ma = None return the_ma @@ -956,7 +1180,8 @@ def geogig(self): return self.get_queryset().filter(name__icontains='geogig') def ows(self): - return self.get_queryset().filter(link_type__in=['OGC:WMS', 'OGC:WFS', 'OGC:WCS']) + return self.get_queryset().filter( + link_type__in=['OGC:WMS', 'OGC:WFS', 'OGC:WCS']) class Link(models.Model): @@ -975,10 +1200,16 @@ class Link(models.Model): * OGC:WCS: for WCS service links """ resource = models.ForeignKey(ResourceBase, blank=True, null=True) - extension = models.CharField(max_length=255, help_text=_('For example "kml"')) - link_type = models.CharField(max_length=255, choices=[(x, x) for x in LINK_TYPES]) - name = models.CharField(max_length=255, help_text=_('For example "View in Google Earth"')) - mime = models.CharField(max_length=255, help_text=_('For example "text/xml"')) + extension = models.CharField( + max_length=255, + help_text=_('For example "kml"')) + link_type = models.CharField( + max_length=255, choices=[ + (x, x) for x in LINK_TYPES]) + name = models.CharField(max_length=255, help_text=_( + 'For example "View in Google Earth"')) + mime = models.CharField(max_length=255, + help_text=_('For example "text/xml"')) url = models.TextField(max_length=1000) objects = LinkManager() @@ -1006,7 +1237,7 @@ def resourcebase_post_save(instance, *args, **kwargs): thumbnail_url=instance.get_thumbnail_url(), detail_url=instance.get_absolute_url(), csw_insert_date=datetime.datetime.now()) - except: + except BaseException: pass try: @@ -1054,16 +1285,17 @@ def resourcebase_post_save(instance, *args, **kwargs): regions_to_add.append(region) if region.level == 0 and region.parent is None: global_regions.append(region) - except: + except BaseException: tb = traceback.format_exc() if tb: logger.debug(tb) if regions_to_add or global_regions: - if regions_to_add and len(regions_to_add) > 0 and len(regions_to_add) <= 30: + if regions_to_add and len( + regions_to_add) > 0 and len(regions_to_add) <= 30: instance.regions.add(*regions_to_add) else: instance.regions.add(*global_regions) - except: + except BaseException: tb = traceback.format_exc() if tb: logger.debug(tb) @@ -1081,7 +1313,9 @@ def rating_post_save(instance, *args, **kwargs): """ Used to fill the average rating field on OverallRating change. """ - ResourceBase.objects.filter(id=instance.object_id).update(rating=instance.rating) + ResourceBase.objects.filter( + id=instance.object_id).update( + rating=instance.rating) signals.post_save.connect(rating_post_save, sender=OverallRating) @@ -1101,11 +1335,14 @@ def do_login(sender, user, request, **kwargs): # Lets create a new one token = generate_token() - AccessToken.objects.get_or_create(user=user, - application=app, - expires=datetime.datetime.now() + datetime.timedelta(days=1), - token=token) - except: + AccessToken.objects.get_or_create( + user=user, + application=app, + expires=datetime.datetime.now() + + datetime.timedelta( + days=1), + token=token) + except BaseException: u = uuid.uuid1() token = u.hex @@ -1123,7 +1360,7 @@ def do_login(sender, user, request, **kwargs): for c in cj: if c.name == "JSESSIONID": jsessionid = c.value - except: + except BaseException: u = uuid.uuid1() jsessionid = u.hex @@ -1144,11 +1381,11 @@ def do_logout(sender, user, request, **kwargs): # Lets delete the old one try: old = AccessToken.objects.get(user=user, application=app) - except: + except BaseException: pass else: old.delete() - except: + except BaseException: pass # Do GeoServer Logout @@ -1186,14 +1423,15 @@ def do_logout(sender, user, request, **kwargs): if cookies: if 'JSESSIONID' in request.session and request.session['JSESSIONID']: - cookies = cookies + '; JSESSIONID=' + request.session['JSESSIONID'] + cookies = cookies + '; JSESSIONID=' + \ + request.session['JSESSIONID'] header_params['Cookie'] = cookies gs_request = urllib2.Request(url, data, header_params) try: urllib2.urlopen(gs_request) - except: + except BaseException: tb = traceback.format_exc() if tb: logger.debug(tb) diff --git a/geonode/catalogue/backends/generic.py b/geonode/catalogue/backends/generic.py index 90c63015af9..2c2c6c845b1 100644 --- a/geonode/catalogue/backends/generic.py +++ b/geonode/catalogue/backends/generic.py @@ -34,13 +34,25 @@ logger = logging.getLogger(__name__) TIMEOUT = 10 -METADATA_FORMATS = {'Atom': ('atom:entry', 'http://www.w3.org/2005/Atom'), - 'DIF': ('dif:DIF', 'http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/'), - 'Dublin Core': ('csw:Record', 'http://www.opengis.net/cat/csw/2.0.2'), - 'ebRIM': ('rim:RegistryObject', 'urn:oasis:names:tc:ebxml-regrep:xsd:rim:3.0'), - 'FGDC': ('fgdc:metadata', 'http://www.opengis.net/cat/csw/csdgm'), - 'ISO': ('gmd:MD_Metadata', 'http://www.isotc211.org/2005/gmd') - } +METADATA_FORMATS = { + 'Atom': ( + 'atom:entry', + 'http://www.w3.org/2005/Atom'), + 'DIF': ( + 'dif:DIF', + 'http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/'), + 'Dublin Core': ( + 'csw:Record', + 'http://www.opengis.net/cat/csw/2.0.2'), + 'ebRIM': ( + 'rim:RegistryObject', + 'urn:oasis:names:tc:ebxml-regrep:xsd:rim:3.0'), + 'FGDC': ( + 'fgdc:metadata', + 'http://www.opengis.net/cat/csw/csdgm'), + 'ISO': ( + 'gmd:MD_Metadata', + 'http://www.isotc211.org/2005/gmd')} class Catalogue(CatalogueServiceWeb): @@ -85,7 +97,9 @@ def login(self): "password": self.password }) request = urllib2.Request(url, post, headers) - self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(), urllib2.HTTPRedirectHandler()) + self.opener = urllib2.build_opener( + urllib2.HTTPCookieProcessor(), + urllib2.HTTPRedirectHandler()) response = self.opener.open(request) doc = etree.fromstring(response.read()) assert doc.tag == 'ok', "GeoNetwork login failed!" @@ -101,7 +115,7 @@ def logout(self): def get_by_uuid(self, uuid): try: self.getrecordbyid([uuid], outputschema=namespaces["gmd"]) - except: + except BaseException: return None if hasattr(self, 'records'): @@ -109,7 +123,11 @@ def get_by_uuid(self, uuid): return None record = self.records.values()[0] record.keywords = [] - if hasattr(record, 'identification') and hasattr(record.identification, 'keywords'): + if hasattr( + record, + 'identification') and hasattr( + record.identification, + 'keywords'): for kw in record.identification.keywords: record.keywords.extend(kw['keywords']) return record @@ -131,7 +149,12 @@ def urls_for_uuid(self, uuid): urls = [] for mformat in self.formats: - urls.append(('text/xml', mformat, self.url_for_uuid(uuid, METADATA_FORMATS[mformat][1]))) + urls.append( + ('text/xml', + mformat, + self.url_for_uuid( + uuid, + METADATA_FORMATS[mformat][1]))) return urls def csw_gen_xml(self, layer, template): @@ -140,12 +163,13 @@ def csw_gen_xml(self, layer, template): id_pname = 'apiso:Identifier' tpl = get_template(template) - ctx = Context({ - 'layer': layer, - 'SITEURL': settings.SITEURL[:-1], - 'id_pname': id_pname, - 'LICENSES_METADATA': getattr(settings, 'LICENSES', dict()).get('METADATA', 'never') - }) + ctx = Context({'layer': layer, + 'SITEURL': settings.SITEURL[:-1], + 'id_pname': id_pname, + 'LICENSES_METADATA': getattr(settings, + 'LICENSES', + dict()).get('METADATA', + 'never')}) md_doc = tpl.render(ctx) return md_doc @@ -179,8 +203,9 @@ def create_from_layer(self, layer): # this is needed for inserting FGDC metadata in GN exml = etree.fromstring(response.read()) - identifier = exml.find('{%s}InsertResult/{%s}BriefRecord/identifier' - % (namespaces['csw'], namespaces['csw'])).text + identifier = exml.find( + '{%s}InsertResult/{%s}BriefRecord/identifier' % + (namespaces['csw'], namespaces['csw'])).text layer.uuid = identifier # Turn on the "view" permission (aka publish) for @@ -188,7 +213,7 @@ def create_from_layer(self, layer): # will be searchable via CSW without admin login. # all other privileges are set to False for all # groups. - self.set_metadata_privs(layer.uuid, {"all": {"view": True}}) + self.set_metadata_privs(layer.uuid, {"all": {"view": True}}) return self.url_for_uuid(layer.uuid, namespaces['gmd']) @@ -232,7 +257,8 @@ def set_metadata_privs(self, uuid, privileges): request = urllib2.Request(get_dbid_url) response = self.urlopen(request) doc = etree.fromstring(response.read()) - data_dbid = doc.find('metadata/{http://www.fao.org/geonetwork}info/id').text + data_dbid = doc.find( + 'metadata/{http://www.fao.org/geonetwork}info/id').text # update group and operation info if needed if len(self._group_ids) == 0: @@ -253,7 +279,8 @@ def set_metadata_privs(self, uuid, privileges): priv_params['_%s_%s' % (group_id, op_id)] = 'on' # update all privileges - update_privs_url = "%sgeonetwork/srv/en/metadata.admin?%s" % (self.base, urllib.urlencode(priv_params)) + update_privs_url = "%sgeonetwork/srv/en/metadata.admin?%s" % ( + self.base, urllib.urlencode(priv_params)) request = urllib2.Request(update_privs_url) response = self.urlopen(request) @@ -265,7 +292,8 @@ def _geonetwork_get_group_ids(self): groups. """ # get the ids of the groups. - get_groups_url = "%sgeonetwork/srv/en/xml.info?%s" % (self.base, urllib.urlencode({'type': 'groups'})) + get_groups_url = "%sgeonetwork/srv/en/xml.info?%s" % ( + self.base, urllib.urlencode({'type': 'groups'})) request = urllib2.Request(get_groups_url) response = self.urlopen(request) doc = etree.fromstring(response.read()) @@ -280,7 +308,8 @@ def _geonetwork_get_operation_ids(self): 'operations' (privileges) """ # get the ids of the operations - get_ops_url = "%sgeonetwork/srv/en/xml.info?%s" % (self.base, urllib.urlencode({'type': 'operations'})) + get_ops_url = "%sgeonetwork/srv/en/xml.info?%s" % ( + self.base, urllib.urlencode({'type': 'operations'})) request = urllib2.Request(get_ops_url) response = self.urlopen(request) doc = etree.fromstring(response.read()) @@ -352,14 +381,19 @@ def metadatarecord2dict(self, rec): 'maxx': rec.identification.bbox.maxx, 'miny': rec.identification.bbox.miny, 'maxy': rec.identification.bbox.maxy - } + } # locate all distribution links result['download_links'] = self.extract_links(rec) - # construct the link to the Catalogue metadata record (not self-indexed) - result['metadata_links'] = [("text/xml", "ISO", self.url_for_uuid(rec.identifier, - 'http://www.isotc211.org/2005/gmd'))] + # construct the link to the Catalogue metadata record (not + # self-indexed) + result['metadata_links'] = [ + ("text/xml", + "ISO", + self.url_for_uuid( + rec.identifier, + 'http://www.isotc211.org/2005/gmd'))] return result @@ -382,7 +416,7 @@ def extract_links(self, rec): format = format_re.match(link_el.description).groups()[0] href = link_el.url links.append((extension, format, href)) - except: + except BaseException: pass return links @@ -403,10 +437,12 @@ def get_record(self, uuid): def search_records(self, keywords, start, limit, bbox): with self.catalogue: bbox = self.catalogue.normalize_bbox(bbox) - self.catalogue.search(keywords, start+1, limit, bbox) + self.catalogue.search(keywords, start + 1, limit, bbox) # build results into JSON for API - results = [self.catalogue.metadatarecord2dict(doc) for v, doc in self.catalogue.records.iteritems()] + results = [ + self.catalogue.metadatarecord2dict(doc) for v, + doc in self.catalogue.records.iteritems()] result = {'rows': results, 'total': self.catalogue.results['matches'], @@ -424,8 +460,9 @@ def remove_record(self, uuid): # model but it just passes it to a Django template so a dict works # too. self.catalogue.delete_layer({"uuid": uuid}) - except: - logger.exception('Couldn\'t delete Catalogue record during cleanup()') + except BaseException: + logger.exception( + 'Couldn\'t delete Catalogue record during cleanup()') def create_record(self, item): with self.catalogue: diff --git a/geonode/contrib/favorite/models.py b/geonode/contrib/favorite/models.py index fc901e464ea..45a5a9e5f36 100755 --- a/geonode/contrib/favorite/models.py +++ b/geonode/contrib/favorite/models.py @@ -35,7 +35,8 @@ def favorites_for_user(self, user): def _favorite_ct_for_user(self, user, model): content_type = ContentType.objects.get_for_model(model) - return self.favorites_for_user(user).filter(content_type=content_type).prefetch_related('content_object') + return self.favorites_for_user(user).filter( + content_type=content_type).prefetch_related('content_object') def favorite_documents_for_user(self, user): return self._favorite_ct_for_user(user, Document) @@ -56,7 +57,10 @@ def favorite_for_user_and_content_object(self, user, content_object): impl note: can only be 0 or 1, per the class's unique_together. """ content_type = ContentType.objects.get_for_model(type(content_object)) - result = self.filter(user=user, content_type=content_type, object_id=content_object.pk) + result = self.filter( + user=user, + content_type=content_type, + object_id=content_object.pk) if len(result) > 0: return result[0] @@ -78,7 +82,7 @@ def create_favorite(self, content_object, user): user=user, content_type=content_type, object_id=content_object.pk, - ) + ) return favorite @@ -98,4 +102,5 @@ class Meta: unique_together = (('user', 'content_type', 'object_id'),) def __unicode__(self): - return "Favorite: {}, {}, {}".format(self.content_object.title, self.content_type, self.user) + return "Favorite: {}, {}, {}".format( + self.content_object.title, self.content_type, self.user) diff --git a/geonode/contrib/mp/signals.py b/geonode/contrib/mp/signals.py index 5c93d3cbb48..8d4a124e72a 100644 --- a/geonode/contrib/mp/signals.py +++ b/geonode/contrib/mp/signals.py @@ -23,9 +23,11 @@ def tileset_post_save(instance, sender, **kwargs): uuid=layer_uuid) if settings.USE_DISK_CACHE: - tile_url = '%s%s/%s/{z}/{x}/{y}.png' % (settings.SITEURL, TILESET_CACHE_URL, instance.id) + tile_url = '%s%s/%s/{z}/{x}/{y}.png' % ( + settings.SITEURL, TILESET_CACHE_URL, instance.id) else: - tile_url = "%sdjmp/%d/map/tiles/%s/EPSG3857/{z}/{x}/{y}.png" % (settings.SITEURL, instance.id, instance.name) + tile_url = "%sdjmp/%d/map/tiles/%s/EPSG3857/{z}/{x}/{y}.png" % ( + settings.SITEURL, instance.id, instance.name) l, __ = Link.objects.get_or_create( resource=layer.resourcebase_ptr, @@ -57,7 +59,7 @@ def layer_post_save(instance, sender, **kwargs): layer_zoom_stop=settings.CACHE_ZOOM_STOP, layer_zoom_start=settings.CACHE_ZOOM_START, layer_uuid=instance.uuid - ) + ) if settings.CACHE_ON_LAYER_LOAD: tileset.seed() diff --git a/geonode/contrib/ows_api/urls.py b/geonode/contrib/ows_api/urls.py index 7ccf4d6b1de..413fb484dd4 100644 --- a/geonode/contrib/ows_api/urls.py +++ b/geonode/contrib/ows_api/urls.py @@ -25,4 +25,4 @@ urlpatterns = [ url(r'^api/ows_endpoints/', ows_endpoints), - ] +] diff --git a/geonode/documents/views.py b/geonode/documents/views.py index 7adfc35ce7b..cfdb8f767a7 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -84,7 +84,7 @@ def document_detail(request, docid): loader.render_to_string( '404.html', RequestContext( request, { - })), status=404) + })), status=404) except PermissionDenied: return HttpResponse( @@ -106,13 +106,17 @@ def document_detail(request, docid): # Update count for popularity ranking, # but do not includes admins or resource owners if request.user != document.owner and not request.user.is_superuser: - Document.objects.filter(id=document.id).update(popular_count=F('popular_count') + 1) + Document.objects.filter( + id=document.id).update( + popular_count=F('popular_count') + 1) metadata = document.link_set.metadata().filter( name__in=settings.DOWNLOAD_FORMATS_METADATA) context_dict = { - 'perms_list': get_perms(request.user, document.get_self_resource()), + 'perms_list': get_perms( + request.user, + document.get_self_resource()), 'permissions_json': _perms_info_json(document), 'resource': document, 'metadata': metadata, @@ -120,7 +124,8 @@ def document_detail(request, docid): 'related': related} if settings.SOCIAL_ORIGINS: - context_dict["social_links"] = build_social_links(request, document) + context_dict["social_links"] = build_social_links( + request, document) if getattr(settings, 'EXIF_ENABLED', False): try: @@ -128,7 +133,7 @@ def document_detail(request, docid): exif = exif_extract_dict(document) if exif: context_dict['exif_data'] = exif - except: + except BaseException: print "Exif extraction failed." return render_to_response( @@ -183,8 +188,10 @@ def form_valid(self, form): # by default, if RESOURCE_PUBLISHING=True then document.is_published # must be set to False # RESOURCE_PUBLISHING works in similar way as ADMIN_MODERATE_UPLOADS, - # but is applied to documents only. ADMIN_MODERATE_UPLOADS has wider usage - is_published = not (settings.RESOURCE_PUBLISHING or settings.ADMIN_MODERATE_UPLOADS) + # but is applied to documents only. ADMIN_MODERATE_UPLOADS has wider + # usage + is_published = not ( + settings.RESOURCE_PUBLISHING or settings.ADMIN_MODERATE_UPLOADS) self.object.is_published = is_published self.object.save() form.save_many2many() @@ -207,7 +214,7 @@ def form_valid(self, form): keywords.extend(exif_metadata.get('keywords', [])) bbox = exif_metadata.get('bbox', None) abstract = exif_metadata.get('abstract', None) - except: + except BaseException: print "Exif extraction failed." if getattr(settings, 'NLP_ENABLED', False): @@ -217,7 +224,7 @@ def form_valid(self, form): if nlp_metadata: regions.extend(nlp_metadata.get('regions', [])) keywords.extend(nlp_metadata.get('keywords', [])) - except: + except BaseException: print "NLP extraction failed." if abstract: @@ -246,8 +253,10 @@ def form_valid(self, form): if getattr(settings, 'SLACK_ENABLED', False): try: from geonode.contrib.slack.utils import build_slack_message_document, send_slack_message - send_slack_message(build_slack_message_document("document_new", self.object)) - except: + send_slack_message( + build_slack_message_document( + "document_new", self.object)) + except BaseException: print "Could not send slack message for new document." if self.request.REQUEST.get('no__redirect', False): @@ -318,7 +327,7 @@ def document_metadata( loader.render_to_string( '404.html', RequestContext( request, { - })), status=404) + })), status=404) except PermissionDenied: return HttpResponse( @@ -344,11 +353,8 @@ def document_metadata( request.POST, instance=document, prefix="resource") - category_form = CategoryForm( - request.POST, - prefix="category_choice_field", - initial=int( - request.POST["category_choice_field"]) if "category_choice_field" in request.POST else None) + category_form = CategoryForm(request.POST, prefix="category_choice_field", initial=int( + request.POST["category_choice_field"]) if "category_choice_field" in request.POST else None) else: document_form = DocumentForm(instance=document, prefix="resource") category_form = CategoryForm( @@ -374,8 +380,10 @@ def document_metadata( if poc_form.is_valid(): if len(poc_form.cleaned_data['profile']) == 0: # FIXME use form.add_error in django > 1.7 - errors = poc_form._errors.setdefault('profile', ErrorList()) - errors.append(_('You must set a point of contact for this resource')) + errors = poc_form._errors.setdefault( + 'profile', ErrorList()) + errors.append( + _('You must set a point of contact for this resource')) poc = None if poc_form.has_changed and poc_form.is_valid(): new_poc = poc_form.save() @@ -389,8 +397,10 @@ def document_metadata( if author_form.is_valid(): if len(author_form.cleaned_data['profile']) == 0: # FIXME use form.add_error in django > 1.7 - errors = author_form._errors.setdefault('profile', ErrorList()) - errors.append(_('You must set an author for this resource')) + errors = author_form._errors.setdefault( + 'profile', ErrorList()) + errors.append( + _('You must set an author for this resource')) metadata_author = None if author_form.has_changed and author_form.is_valid(): new_author = author_form.save() @@ -401,13 +411,17 @@ def document_metadata( the_document.metadata_author = new_author the_document.keywords.add(*new_keywords) document_form.save_many2many() - Document.objects.filter(id=the_document.id).update(category=new_category) + Document.objects.filter( + id=the_document.id).update( + category=new_category) if getattr(settings, 'SLACK_ENABLED', False): try: from geonode.contrib.slack.utils import build_slack_message_document, send_slack_messages - send_slack_messages(build_slack_message_document("document_edit", the_document)) - except: + send_slack_messages( + build_slack_message_document( + "document_edit", the_document)) + except BaseException: print "Could not send slack message for modified document." return HttpResponseRedirect( @@ -432,7 +446,9 @@ def document_metadata( metadata_author_groups = GroupProfile.objects.all() else: metadata_author_groups = chain( - metadata_author.group_list_all(), GroupProfile.objects.exclude(access="private")) + metadata_author.group_list_all(), + GroupProfile.objects.exclude( + access="private")) return render_to_response(template, RequestContext(request, { "resource": document, @@ -449,9 +465,9 @@ def document_metadata( @login_required def document_metadata_advanced(request, docid): return document_metadata( - request, - docid, - template='documents/document_metadata_advanced.html') + request, + docid, + template='documents/document_metadata_advanced.html') def document_search_page(request): @@ -494,8 +510,9 @@ def document_remove(request, docid, template='documents/document_remove.html'): slack_message = None try: from geonode.contrib.slack.utils import build_slack_message_document - slack_message = build_slack_message_document("document_delete", document) - except: + slack_message = build_slack_message_document( + "document_delete", document) + except BaseException: print "Could not build slack message for delete document." document.delete() @@ -503,7 +520,7 @@ def document_remove(request, docid, template='documents/document_remove.html'): try: from geonode.contrib.slack.utils import send_slack_messages send_slack_messages(slack_message) - except: + except BaseException: print "Could not send slack message for delete document." else: document.delete() @@ -520,7 +537,10 @@ def document_remove(request, docid, template='documents/document_remove.html'): ) -def document_metadata_detail(request, docid, template='documents/document_metadata_detail.html'): +def document_metadata_detail( + request, + docid, + template='documents/document_metadata_detail.html'): document = _resolve_document( request, docid, diff --git a/geonode/layers/models.py b/geonode/layers/models.py index ccbb9b6199b..50277db02b4 100644 --- a/geonode/layers/models.py +++ b/geonode/layers/models.py @@ -79,14 +79,19 @@ def __str__(self): def absolute_url(self): if self.sld_url: - if self.sld_url.startswith(settings.OGC_SERVER['default']['LOCATION']): - return self.sld_url.split(settings.OGC_SERVER['default']['LOCATION'], 1)[1] + if self.sld_url.startswith( + settings.OGC_SERVER['default']['LOCATION']): + return self.sld_url.split( + settings.OGC_SERVER['default']['LOCATION'], 1)[1] elif self.sld_url.startswith(settings.OGC_SERVER['default']['PUBLIC_LOCATION']): - return self.sld_url.split(settings.OGC_SERVER['default']['PUBLIC_LOCATION'], 1)[1] + return self.sld_url.split( + settings.OGC_SERVER['default']['PUBLIC_LOCATION'], 1)[1] return self.sld_url else: - logger.error("SLD URL is empty for Style %s" % self.name.encode('utf-8')) + logger.error( + "SLD URL is empty for Style %s" % + self.name.encode('utf-8')) return None @@ -113,7 +118,11 @@ class Layer(ResourceBase): is_mosaic = models.BooleanField(default=False) has_time = models.BooleanField(default=False) has_elevation = models.BooleanField(default=False) - time_regex = models.CharField(max_length=128, null=True, blank=True, choices=TIME_REGEX) + time_regex = models.CharField( + max_length=128, + null=True, + blank=True, + choices=TIME_REGEX) elevation_regex = models.CharField(max_length=128, null=True, blank=True) default_style = models.ForeignKey( @@ -225,7 +234,8 @@ def get_base_file(self): # we need to check, for shapefile, if column names are valid list_col = None if self.storeType == 'dataStore': - valid_shp, wrong_column_name, list_col = check_shp_columnnames(self) + valid_shp, wrong_column_name, list_col = check_shp_columnnames( + self) if wrong_column_name: msg = 'Shapefile has an invalid column name: %s' % wrong_column_name else: @@ -286,7 +296,11 @@ def geogig_enabled(self): @property def geogig_link(self): if(self.geogig_enabled): - return getattr(self.link_set.filter(name__icontains='clone in geogig').first(), 'url', None) + return getattr( + self.link_set.filter( + name__icontains='clone in geogig').first(), + 'url', + None) return None @@ -312,8 +326,11 @@ class LayerFile(models.Model): upload_session = models.ForeignKey(UploadSession) name = models.CharField(max_length=255) base = models.BooleanField(default=False) - file = models.FileField(upload_to='layers', - storage=FileSystemStorage(base_url=settings.LOCAL_MEDIA_URL), max_length=255) + file = models.FileField( + upload_to='layers', + storage=FileSystemStorage( + base_url=settings.LOCAL_MEDIA_URL), + max_length=255) class AttributeManager(models.Manager): @@ -373,10 +390,8 @@ class Attribute(models.Model): _('visible?'), help_text=_('specifies if the attribute should be displayed in identify results'), default=True) - display_order = models.IntegerField( - _('display order'), - help_text=_('specifies the order in which attribute should be displayed in identify results'), - default=1) + display_order = models.IntegerField(_('display order'), help_text=_( + 'specifies the order in which attribute should be displayed in identify results'), default=1) # statistical derivations count = models.IntegerField( @@ -436,10 +451,8 @@ class Attribute(models.Model): null=True, blank=True, default='NA') - last_stats_updated = models.DateTimeField( - _('last modified'), - default=datetime.now, - help_text=_('date when attribute statistics were last updated')) # passing the method itself, not + last_stats_updated = models.DateTimeField(_('last modified'), default=datetime.now, help_text=_( + 'date when attribute statistics were last updated')) # passing the method itself, not objects = AttributeManager() diff --git a/geonode/layers/utils.py b/geonode/layers/utils.py index 4ce04931f2b..2f8dff6681d 100644 --- a/geonode/layers/utils.py +++ b/geonode/layers/utils.py @@ -84,7 +84,8 @@ def resolve_regions(regions): if len(regions) > 0: for region in regions: try: - region_resolved = Region.objects.get(Q(name__iexact=region) | Q(code__iexact=region)) + region_resolved = Region.objects.get( + Q(name__iexact=region) | Q(code__iexact=region)) regions_resolved.append(region_resolved) except ObjectDoesNotExist: regions_unresolved.append(region) @@ -432,8 +433,10 @@ def file_upload(filename, name=None, user=None, title=None, abstract=None, assigned_name = None for type_name, fn in files.items(): with open(fn, 'rb') as f: - upload_session.layerfile_set.create(name=type_name, - file=File(f, name='%s.%s' % (assigned_name or valid_name, type_name))) + upload_session.layerfile_set.create( + name=type_name, file=File( + f, name='%s.%s' % + (assigned_name or valid_name, type_name))) # save the system assigned name for the remaining files if not assigned_name: the_file = upload_session.layerfile_set.all()[0].file.name @@ -502,7 +505,7 @@ def file_upload(filename, name=None, user=None, title=None, abstract=None, if nlp_metadata: regions_resolved.extend(nlp_metadata.get('regions', [])) keywords.extend(nlp_metadata.get('keywords', [])) - except: + except BaseException: print "NLP extraction failed." # If it is a vector file, create the layer in postgis. @@ -669,38 +672,45 @@ def upload(incoming, user=None, overwrite=False, if tarfile.is_tarfile(filename): filename = extract_tarfile(filename) - layer = file_upload(filename, - name=name, - title=title, - abstract=abstract, - date=date, - user=user, - overwrite=overwrite, - license=license, - category=category, - keywords=keywords, - regions=regions, - metadata_uploaded_preserve=metadata_uploaded_preserve - ) + layer = file_upload( + filename, + name=name, + title=title, + abstract=abstract, + date=date, + user=user, + overwrite=overwrite, + license=license, + category=category, + keywords=keywords, + regions=regions, + metadata_uploaded_preserve=metadata_uploaded_preserve) if not existed: status = 'created' else: status = 'updated' if private and user: - perm_spec = {"users": {"AnonymousUser": [], - user.username: ["change_resourcebase_metadata", "change_layer_data", - "change_layer_style", "change_resourcebase", - "delete_resourcebase", "change_resourcebase_permissions", - "publish_resourcebase"]}, "groups": {}} + perm_spec = { + "users": { + "AnonymousUser": [], + user.username: [ + "change_resourcebase_metadata", + "change_layer_data", + "change_layer_style", + "change_resourcebase", + "delete_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase"]}, + "groups": {}} layer.set_permissions(perm_spec) if getattr(settings, 'SLACK_ENABLED', False): try: from geonode.contrib.slack.utils import build_slack_message_layer, send_slack_messages - send_slack_messages(build_slack_message_layer( - ("layer_new" if status == "created" else "layer_edit"), - layer)) - except: + send_slack_messages( + build_slack_message_layer( + ("layer_new" if status == "created" else "layer_edit"), layer)) + except BaseException: print "Could not send slack message." except Exception as e: @@ -771,8 +781,8 @@ def create_thumbnail(instance, thumbnail_remote_url, thumbnail_create_url=None, name="Remote Thumbnail", mime='image/png', link_type='image', - ) - ) + ) + ) Layer.objects.filter(id=instance.id) \ .update(thumbnail_url=thumbnail_remote_url) # Download thumbnail and save it locally. diff --git a/geonode/layers/views.py b/geonode/layers/views.py index 0378c12e767..a4d31a2c45b 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -201,6 +201,13 @@ def layer_upload(request, template='upload/layer_upload.html'): out['url'] = reverse( 'layer_detail', args=[ saved_layer.service_typename]) + if hasattr(saved_layer, 'bbox_string'): + out['bbox'] = saved_layer.bbox_string + if hasattr(saved_layer, 'srid'): + out['crs'] = { + 'type': 'name', + 'properties': saved_layer.srid + } upload_session = saved_layer.upload_session upload_session.processed = True upload_session.save() diff --git a/geonode/maps/models.py b/geonode/maps/models.py index bb128db717e..d8a35c50c64 100644 --- a/geonode/maps/models.py +++ b/geonode/maps/models.py @@ -316,7 +316,7 @@ def layer_group(self): return { 'catalog': gs_catalog.get_layergroup(lg_name), 'ows': ogc_server_settings.ows - } + } else: return None @@ -475,7 +475,7 @@ def layer_config(self, user=None): obj=layer.resourcebase_ptr): cfg['disabled'] = True cfg['visibility'] = False - except: + except BaseException: # shows maplayer with pink tiles, # and signals that there is problem # TODO: clear orphaned MapLayers diff --git a/geonode/maps/views.py b/geonode/maps/views.py index 2bca236f324..2a491022dd5 100644 --- a/geonode/maps/views.py +++ b/geonode/maps/views.py @@ -114,12 +114,18 @@ def map_detail(request, mapid, snapshot=None, template='maps/map_detail.html'): The view that show details of each map ''' - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) # Update count for popularity ranking, # but do not includes admins or resource owners if request.user != map_obj.owner and not request.user.is_superuser: - Map.objects.filter(id=map_obj.id).update(popular_count=F('popular_count') + 1) + Map.objects.filter( + id=map_obj.id).update( + popular_count=F('popular_count') + 1) if 'access_token' in request.session: access_token = request.session['access_token'] @@ -161,7 +167,11 @@ def map_detail(request, mapid, snapshot=None, template='maps/map_detail.html'): @login_required def map_metadata(request, mapid, template='maps/map_metadata.html'): - map_obj = _resolve_map(request, mapid, 'base.change_resourcebase_metadata', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.change_resourcebase_metadata', + _PERMISSION_MSG_VIEW) poc = map_obj.poc @@ -171,11 +181,8 @@ def map_metadata(request, mapid, template='maps/map_metadata.html'): if request.method == "POST": map_form = MapForm(request.POST, instance=map_obj, prefix="resource") - category_form = CategoryForm( - request.POST, - prefix="category_choice_field", - initial=int( - request.POST["category_choice_field"]) if "category_choice_field" in request.POST else None) + category_form = CategoryForm(request.POST, prefix="category_choice_field", initial=int( + request.POST["category_choice_field"]) if "category_choice_field" in request.POST else None) else: map_form = MapForm(instance=map_obj, prefix="resource") category_form = CategoryForm( @@ -227,8 +234,10 @@ def map_metadata(request, mapid, template='maps/map_metadata.html'): if getattr(settings, 'SLACK_ENABLED', False): try: from geonode.contrib.slack.utils import build_slack_message_map, send_slack_messages - send_slack_messages(build_slack_message_map("map_edit", the_map)) - except: + send_slack_messages( + build_slack_message_map( + "map_edit", the_map)) + except BaseException: print "Could not send slack message for modified map." return HttpResponseRedirect( @@ -272,7 +281,10 @@ def map_metadata(request, mapid, template='maps/map_metadata.html'): if request.user.is_superuser: metadata_author_groups = GroupProfile.objects.all() else: - metadata_author_groups = chain(metadata_author.group_list_all(), GroupProfile.objects.exclude(access="private")) + metadata_author_groups = chain( + metadata_author.group_list_all(), + GroupProfile.objects.exclude( + access="private")) return render_to_response(template, RequestContext(request, { "config": json.dumps(config), @@ -283,8 +295,8 @@ def map_metadata(request, mapid, template='maps/map_metadata.html'): "author_form": author_form, "category_form": category_form, "layers": layers, - "preview": getattr(settings, 'LAYER_PREVIEW_LIBRARY', 'leaflet'), - "crs": getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:900913'), + "preview": getattr(settings, 'LAYER_PREVIEW_LIBRARY', 'leaflet'), + "crs": getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:900913'), "metadata_author_groups": metadata_author_groups, "GROUP_MANDATORY_RESOURCES": getattr(settings, 'GROUP_MANDATORY_RESOURCES', False), })) @@ -292,13 +304,20 @@ def map_metadata(request, mapid, template='maps/map_metadata.html'): @login_required def map_metadata_advanced(request, mapid): - return map_metadata(request, mapid, template='maps/map_metadata_advanced.html') + return map_metadata( + request, + mapid, + template='maps/map_metadata_advanced.html') @login_required def map_remove(request, mapid, template='maps/map_remove.html'): ''' Delete a map, and its constituent layers. ''' - map_obj = _resolve_map(request, mapid, 'base.delete_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.delete_resourcebase', + _PERMISSION_MSG_VIEW) if request.method == 'GET': return render_to_response(template, RequestContext(request, { @@ -313,7 +332,7 @@ def map_remove(request, mapid, template='maps/map_remove.html'): try: from geonode.contrib.slack.utils import build_slack_message_map slack_message = build_slack_message_map("map_delete", map_obj) - except: + except BaseException: print "Could not build slack message for delete map." delete_map.delay(object_id=map_obj.id) @@ -321,7 +340,7 @@ def map_remove(request, mapid, template='maps/map_remove.html'): try: from geonode.contrib.slack.utils import send_slack_messages send_slack_messages(slack_message) - except: + except BaseException: print "Could not send slack message for delete map." else: @@ -339,7 +358,11 @@ def map_embed( if mapid is None: config = default_map_config(request)[0] else: - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) if 'access_token' in request.session: access_token = request.session['access_token'] @@ -349,7 +372,8 @@ def map_embed( if snapshot is None: config = map_obj.viewer_json(request.user, access_token) else: - config = snapshot_config(snapshot, map_obj, request.user, access_token) + config = snapshot_config( + snapshot, map_obj, request.user, access_token) return render_to_response(template, RequestContext(request, { 'config': json.dumps(config) @@ -364,7 +388,11 @@ def map_view(request, mapid, snapshot=None, template='maps/map_view.html'): The view that returns the map composer opened to the map with the given map ID. """ - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) if 'access_token' in request.session: access_token = request.session['access_token'] @@ -387,25 +415,39 @@ def map_view(request, mapid, snapshot=None, template='maps/map_view.html'): def map_view_js(request, mapid): - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) if 'access_token' in request.session: access_token = request.session['access_token'] else: access_token = None config = map_obj.viewer_json(request.user, access_token) - return HttpResponse(json.dumps(config), content_type="application/javascript") + return HttpResponse( + json.dumps(config), + content_type="application/javascript") def map_json(request, mapid, snapshot=None): if request.method == 'GET': - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) if 'access_token' in request.session: access_token = request.session['access_token'] else: access_token = None - return HttpResponse(json.dumps(map_obj.viewer_json(request.user, access_token))) + return HttpResponse( + json.dumps( + map_obj.viewer_json( + request.user, + access_token))) elif request.method == 'PUT': if not request.user.is_authenticated(): return HttpResponse( @@ -415,7 +457,9 @@ def map_json(request, mapid, snapshot=None): ) map_obj = Map.objects.get(id=mapid) - if not request.user.has_perm('change_resourcebase', map_obj.get_self_resource()): + if not request.user.has_perm( + 'change_resourcebase', + map_obj.get_self_resource()): return HttpResponse( _PERMISSION_MSG_SAVE, status=401, @@ -434,7 +478,11 @@ def map_json(request, mapid, snapshot=None): else: access_token = None - return HttpResponse(json.dumps(map_obj.viewer_json(request.user, access_token))) + return HttpResponse( + json.dumps( + map_obj.viewer_json( + request.user, + access_token))) except ValueError as e: return HttpResponse( "The server could not understand the request." + str(e), @@ -448,7 +496,11 @@ def map_edit(request, mapid, snapshot=None, template='maps/map_edit.html'): The view that returns the map composer opened to the map with the given map ID. """ - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) if 'access_token' in request.session: access_token = request.session['access_token'] @@ -508,7 +560,9 @@ def new_map(request, template='maps/map_new.html'): if isinstance(config, HttpResponse): return config else: - return render_to_response(template, RequestContext(request, context_dict)) + return render_to_response( + template, RequestContext( + request, context_dict)) def new_map_json(request): @@ -598,7 +652,7 @@ def new_map_config(request): if 'layer' in params: bbox = None map_obj = Map(projection=getattr(settings, 'DEFAULT_MAP_CRS', - 'EPSG:900913')) + 'EPSG:900913')) if request.user.is_authenticated(): map_obj.owner = request.user @@ -635,19 +689,22 @@ def new_map_config(request): config["title"] = layer.title config["queryable"] = True - config["srs"] = getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:900913') + config["srs"] = getattr( + settings, 'DEFAULT_MAP_CRS', 'EPSG:900913') config["bbox"] = bbox if config["srs"] != 'EPSG:900913' \ else llbbox_to_mercator([float(coord) for coord in bbox]) if layer.storeType == "remoteStore": service = layer.service # Probably not a good idea to send the access token to every remote service. - # This should never match, so no access token should be sent to remote services. - ogc_server_url = urlparse.urlsplit(ogc_server_settings.PUBLIC_LOCATION).netloc + # This should never match, so no access token should be + # sent to remote services. + ogc_server_url = urlparse.urlsplit( + ogc_server_settings.PUBLIC_LOCATION).netloc service_url = urlparse.urlsplit(service.base_url).netloc if access_token and ogc_server_url == service_url and 'access_token' not in service.base_url: - url = service.base_url+'?access_token='+access_token + url = service.base_url + '?access_token=' + access_token else: url = service.base_url maplayer = MapLayer(map=map_obj, @@ -661,11 +718,12 @@ def new_map_config(request): "url": url, "name": service.name})) else: - ogc_server_url = urlparse.urlsplit(ogc_server_settings.PUBLIC_LOCATION).netloc + ogc_server_url = urlparse.urlsplit( + ogc_server_settings.PUBLIC_LOCATION).netloc layer_url = urlparse.urlsplit(layer.ows_url).netloc if access_token and ogc_server_url == layer_url and 'access_token' not in layer.ows_url: - url = layer.ows_url+'?access_token='+access_token + url = layer.ows_url + '?access_token=' + access_token else: url = layer.ows_url maplayer = MapLayer( @@ -684,7 +742,10 @@ def new_map_config(request): x = (minx + maxx) / 2 y = (miny + maxy) / 2 - if getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:900913') == "EPSG:4326": + if getattr( + settings, + 'DEFAULT_MAP_CRS', + 'EPSG:900913') == "EPSG:4326": center = list((x, y)) else: center = list(forward_mercator((x, y))) @@ -729,7 +790,11 @@ def map_download(request, mapid, template='maps/map_download.html'): XXX To do, remove layer status once progress id done This should be fix because """ - map_obj = _resolve_map(request, mapid, 'base.download_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.download_resourcebase', + _PERMISSION_MSG_VIEW) map_status = dict() if request.method == 'POST': @@ -823,7 +888,11 @@ def map_download_check(request): def map_wmc(request, mapid, template="maps/wmc.xml"): """Serialize an OGC Web Map Context Document (WMC) 1.1""" - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) return render_to_response(template, RequestContext(request, { 'map': map_obj, @@ -840,7 +909,11 @@ def map_wms(request, mapid): GET: return endpoint information for group layer, PUT: update existing or create new group layer. """ - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) if request.method == 'PUT': try: @@ -852,7 +925,7 @@ def map_wms(request, mapid): return HttpResponse( json.dumps(response), content_type="application/json") - except: + except BaseException: return HttpResponseServerError() if request.method == 'GET': @@ -860,7 +933,9 @@ def map_wms(request, mapid): layerGroupName=getattr(map_obj.layer_group, 'name', ''), ows=getattr(ogc_server_settings, 'ows', ''), ) - return HttpResponse(json.dumps(response), content_type="application/json") + return HttpResponse( + json.dumps(response), + content_type="application/json") return HttpResponseNotAllowed(['PUT', 'GET']) @@ -946,7 +1021,10 @@ def featured_map(request, site): The view that returns the map composer opened to the map with the given official site url. """ - map_obj = resolve_object(request, Map, {'featuredurl': site}, permission='base.view_resourcebase', + map_obj = resolve_object(request, + Map, + {'featuredurl': site}, + permission='base.view_resourcebase', permission_msg=_PERMISSION_MSG_VIEW) return map_view(request, str(map_obj.id)) @@ -956,7 +1034,10 @@ def featured_map_info(request, site): main view for map resources, dispatches to correct view based on method and query args. ''' - map_obj = resolve_object(request, Map, {'featuredurl': site}, permission='base.view_resourcebase', + map_obj = resolve_object(request, + Map, + {'featuredurl': site}, + permission='base.view_resourcebase', permission_msg=_PERMISSION_MSG_VIEW) return map_detail(request, str(map_obj.id)) @@ -975,11 +1056,18 @@ def snapshot_create(request): id=config['id'])) return HttpResponse(num_encode(snapshot.id), content_type="text/plain") else: - return HttpResponse("Invalid JSON", content_type="text/plain", status=500) + return HttpResponse( + "Invalid JSON", + content_type="text/plain", + status=500) def ajax_snapshot_history(request, mapid): - map_obj = _resolve_map(request, mapid, 'base.view_resourcebase', _PERMISSION_MSG_VIEW) + map_obj = _resolve_map( + request, + mapid, + 'base.view_resourcebase', + _PERMISSION_MSG_VIEW) history = [snapshot.json() for snapshot in map_obj.snapshots] return HttpResponse(json.dumps(history), content_type="text/plain") @@ -1026,7 +1114,7 @@ def map_thumbnail(request, mapid): map_obj.save_thumbnail(filename, image) return HttpResponse('Thumbnail saved') - except: + except BaseException: return HttpResponse( content='error saving thumbnail', status=500, @@ -1034,7 +1122,10 @@ def map_thumbnail(request, mapid): ) -def map_metadata_detail(request, mapid, template='maps/map_metadata_detail.html'): +def map_metadata_detail( + request, + mapid, + template='maps/map_metadata_detail.html'): map_obj = _resolve_map(request, mapid, 'view_resourcebase') return render_to_response(template, RequestContext(request, { "resource": map_obj, diff --git a/geonode/services/views.py b/geonode/services/views.py index cf76dd50c88..1c691bd9b8d 100644 --- a/geonode/services/views.py +++ b/geonode/services/views.py @@ -85,9 +85,8 @@ def services(request): This view shows the list of all registered services """ services = Service.objects.all() - return render_to_response("services/service_list.html", RequestContext(request, { - 'services': services, - })) + return render_to_response("services/service_list.html", + RequestContext(request, {'services': services, })) @login_required @@ -120,7 +119,8 @@ def register_service(request): type, server = _verify_service_type(url) if type is None: - return HttpResponse('Could not determine server type', status=400) + return HttpResponse( + 'Could not determine server type', status=400) if "user" in request.POST and "password" in request.POST: user = request.POST.get('user') @@ -130,11 +130,14 @@ def register_service(request): password = None if type in ["WMS", "OWS"]: - return _process_wms_service(url, name, type, user, password, wms=server, owner=request.user) + return _process_wms_service( + url, name, type, user, password, wms=server, owner=request.user) elif type == "REST": - return _register_arcgis_url(url, name, user, password, owner=request.user) + return _register_arcgis_url( + url, name, user, password, owner=request.user) elif type == "CSW": - return _register_harvested_service(url, name, user, password, owner=request.user) + return _register_harvested_service( + url, name, user, password, owner=request.user) elif type == "OGP": return _register_ogp_service(url, owner=request.user) else: @@ -188,7 +191,8 @@ def _clean_url(base_url): # hack, we make sure to append the map parameter for MapServer endpoints # that are exposing it if 'map' in urlparse.parse_qs(urlprop.query): - map_param = urllib.urlencode({'map': urlparse.parse_qs(urlprop.query)['map'][0]}) + map_param = urllib.urlencode( + {'map': urlparse.parse_qs(urlprop.query)['map'][0]}) url = '%s?%s' % (url, map_param) return url @@ -218,7 +222,7 @@ def _verify_service_type(base_url, service_type=None): if service_type in ['WMS', 'OWS', None]: try: service = WebMapService(base_url) - except: + except BaseException: pass else: return ['WMS', service] @@ -226,7 +230,7 @@ def _verify_service_type(base_url, service_type=None): if service_type in ['WFS', 'OWS', None]: try: servicewfs = WebFeatureService(base_url) - except: + except BaseException: pass else: return ['WFS', servicewfs] @@ -234,7 +238,7 @@ def _verify_service_type(base_url, service_type=None): if service_type in ['TMS', None]: try: service = TileMapService(base_url) - except: + except BaseException: pass else: return ['TMS', service] @@ -242,7 +246,7 @@ def _verify_service_type(base_url, service_type=None): if service_type in ['REST', None]: try: service = ArcFolder(base_url) - except: + except BaseException: pass else: try: @@ -254,7 +258,7 @@ def _verify_service_type(base_url, service_type=None): if service_type in ['CSW', None]: try: service = CatalogueServiceWeb(base_url) - except Exception, e: + except Exception as e: logger.exception(e) raise else: @@ -268,7 +272,15 @@ def _verify_service_type(base_url, service_type=None): return [None, None] -def _process_wms_service(url, name, type, username, password, wms=None, owner=None, parent=None): +def _process_wms_service( + url, + name, + type, + username, + password, + wms=None, + owner=None, + parent=None): """ Create a new WMS/OWS service, cascade it if necessary (i.e. if Web Mercator not available) """ @@ -281,9 +293,10 @@ def _process_wms_service(url, name, type, username, password, wms=None, owner=No if base_url and base_url != url: url = base_url wms = WebMapService(base_url) - except: + except BaseException: logger.info( - "Could not retrieve GetMap url, using originally supplied URL %s" % url) + "Could not retrieve GetMap url, using originally supplied URL %s" % + url) pass try: service = Service.objects.get(base_url=url) @@ -296,7 +309,7 @@ def _process_wms_service(url, name, type, username, password, wms=None, owner=No return HttpResponse(json.dumps(return_dict), content_type='application/json', status=200) - except: + except BaseException: pass title = wms.identification.title @@ -307,15 +320,41 @@ def _process_wms_service(url, name, type, username, password, wms=None, owner=No name = _get_valid_name(urlsplit(url).netloc) try: supported_crs = ','.join(wms.contents.itervalues().next().crsOptions) - except: + except BaseException: supported_crs = None - if supported_crs and re.search('EPSG:900913|EPSG:3857|EPSG:102100|EPSG:102113', supported_crs): - return _register_indexed_service(type, url, name, username, password, wms=wms, owner=owner, parent=parent) + if supported_crs and re.search( + 'EPSG:900913|EPSG:3857|EPSG:102100|EPSG:102113', + supported_crs): + return _register_indexed_service( + type, + url, + name, + username, + password, + wms=wms, + owner=owner, + parent=parent) else: - return _register_cascaded_service(url, type, name, username, password, wms=wms, owner=owner, parent=parent) - - -def _register_cascaded_service(url, type, name, username, password, wms=None, owner=None, parent=None): + return _register_cascaded_service( + url, + type, + name, + username, + password, + wms=wms, + owner=owner, + parent=parent) + + +def _register_cascaded_service( + url, + type, + name, + username, + password, + wms=None, + owner=None, + parent=None): """ Register a service as cascading WMS """ @@ -328,7 +367,7 @@ def _register_cascaded_service(url, type, name, username, password, wms=None, ow return HttpResponse(json.dumps(return_dict), content_type='application/json', status=200) - except: + except BaseException: # TODO: Handle this error properly pass @@ -363,7 +402,7 @@ def _register_cascaded_service(url, type, name, username, password, wms=None, ow try: cascade_store = cat.get_store(name, cascade_ws) - except: + except BaseException: cascade_store = cat.create_wmsstore( name, cascade_ws, username, password) cascade_store.capabilitiesURL = url @@ -383,7 +422,7 @@ def _register_cascaded_service(url, type, name, username, password, wms=None, ow try: wfs_ds = cat.get_store(name, cascade_ws) - except: + except BaseException: wfs_ds = cat.create_datastore(name, cascade_ws) connection_params = { "WFSDataStoreFactory:MAXFEATURES": "0", @@ -495,9 +534,7 @@ def _register_cascaded_layers(service, owner=None): cascaded_layer.set_default_permissions() service_layer, created = ServiceLayer.objects.get_or_create( - service=service, - typename=cascaded_layer.name - ) + service=service, typename=cascaded_layer.name) service_layer.layer = cascaded_layer service_layer.title = cascaded_layer.title, service_layer.description = cascaded_layer.abstract, @@ -507,7 +544,8 @@ def _register_cascaded_layers(service, owner=None): count += 1 else: logger.error( - "Resource %s from store %s could not be saved as layer" % (layer, store.name)) + "Resource %s from store %s could not be saved as layer" % + (layer, store.name)) message = "%d Layers Registered" % count return_dict = {'status': 'ok', 'msg': message} return HttpResponse(json.dumps(return_dict), @@ -519,7 +557,16 @@ def _register_cascaded_layers(service, owner=None): return HttpResponse('Invalid Service Type', status=400) -def _register_indexed_service(type, url, name, username, password, verbosity=False, wms=None, owner=None, parent=None): +def _register_indexed_service( + type, + url, + name, + username, + password, + verbosity=False, + wms=None, + owner=None, + parent=None): """ Register a service - WMS or OWS currently supported """ @@ -538,20 +585,20 @@ def _register_indexed_service(type, url, name, username, password, verbosity=Fal return HttpResponse(json.dumps(return_dict), content_type='application/json', status=200) - except: + except BaseException: pass - service = Service.objects.create(base_url=url, - type=type, - method='I', - name=name, - version=wms.identification.version, - title=wms.identification.title or name, - abstract=wms.identification.abstract or _( - "Not provided"), - online_resource=wms.provider.url, - owner=owner, - parent=parent) + service = Service.objects.create( + base_url=url, + type=type, + method='I', + name=name, + version=wms.identification.version, + title=wms.identification.title or name, + abstract=wms.identification.abstract or _("Not provided"), + online_resource=wms.provider.url, + owner=owner, + parent=parent) service.keywords = ','.join(wms.identification.keywords) service.save() @@ -607,7 +654,7 @@ def _register_indexed_layers(service, wms=None, verbosity=False): layer_uuid = str(uuid.uuid1()) try: keywords = map(lambda x: x[:100], wms_layer.keywords) - except: + except BaseException: keywords = [] if not wms_layer.abstract: abstract = "" @@ -621,7 +668,8 @@ def _register_indexed_layers(service, wms=None, verbosity=False): srid = 'EPSG:900913' elif len(wms_layer.crsOptions) > 0: matches = re.findall( - 'EPSG\:(3857|102100|102113)', ' '.join(wms_layer.crsOptions)) + 'EPSG\:(3857|102100|102113)', ' '.join( + wms_layer.crsOptions)) if matches: srid = 'EPSG:%s' % matches[0] if srid is None: @@ -642,9 +690,12 @@ def _register_indexed_layers(service, wms=None, verbosity=False): if not existing_layer: try: - signals.post_save.disconnect(resourcebase_post_save, sender=Layer) - signals.post_save.disconnect(catalogue_post_save, sender=Layer) - signals.post_save.disconnect(geoserver_post_save, sender=Layer) + signals.post_save.disconnect( + resourcebase_post_save, sender=Layer) + signals.post_save.disconnect( + catalogue_post_save, sender=Layer) + signals.post_save.disconnect( + geoserver_post_save, sender=Layer) saved_layer = Layer.objects.create( typename=wms_layer.name, name=wms_layer.name, @@ -666,9 +717,7 @@ def _register_indexed_layers(service, wms=None, verbosity=False): saved_layer.keywords.add(*keywords) service_layer, created = ServiceLayer.objects.get_or_create( - typename=wms_layer.name, - service=service - ) + typename=wms_layer.name, service=service) service_layer.layer = saved_layer service_layer.title = wms_layer.title service_layer.description = wms_layer.abstract @@ -683,13 +732,16 @@ def _register_indexed_layers(service, wms=None, verbosity=False): create_wms_links(saved_layer, wms.url) - except Exception, e: + except Exception as e: logger.error("Error registering %s:%s" % (wms_layer.name, str(e))) finally: - signals.post_save.connect(resourcebase_post_save, sender=Layer) - signals.post_save.connect(catalogue_post_save, sender=Layer) - signals.post_save.connect(geoserver_post_save, sender=Layer) + signals.post_save.connect( + resourcebase_post_save, sender=Layer) + signals.post_save.connect( + catalogue_post_save, sender=Layer) + signals.post_save.connect( + geoserver_post_save, sender=Layer) count += 1 message = "%d Layers Registered" % count return_dict = {'status': 'ok', 'msg': message} @@ -704,7 +756,13 @@ def _register_indexed_layers(service, wms=None, verbosity=False): return HttpResponse('Invalid Service Type', status=400) -def _register_harvested_service(url, name, username, password, csw=None, owner=None): +def _register_harvested_service( + url, + name, + username, + password, + csw=None, + owner=None): """ Register a CSW service, then step through results (or queue for asynchronous harvesting) """ @@ -720,21 +778,22 @@ def _register_harvested_service(url, name, username, password, csw=None, owner=N return HttpResponse(json.dumps(return_dict), content_type='application/json', status=200) - except: + except BaseException: pass if csw is None: csw = CatalogueServiceWeb(url) - service = Service.objects.create(base_url=url, - type='CSW', - method='H', - name=_get_valid_name( - csw.identification.title or url) if not name else name, - title=csw.identification.title, - version=csw.identification.version, - abstract=csw.identification.abstract or _("Not provided"), - owner=owner) + service = Service.objects.create( + base_url=url, + type='CSW', + method='H', + name=_get_valid_name( + csw.identification.title or url) if not name else name, + title=csw.identification.title, + version=csw.identification.version, + abstract=csw.identification.abstract or _("Not provided"), + owner=owner) service.keywords = ','.join(csw.identification.keywords) service.save @@ -779,9 +838,8 @@ def _harvest_csw(csw, maxrecords=10, totalrecords=float('inf')): max = min(src.results['matches'], totalrecords) - if src.results['nextrecord'] == 0 \ - or src.results['returned'] == 0 \ - or src.results['nextrecord'] > max: # end the loop, exhausted all records or max records to process + # end the loop, exhausted all records or max records to process + if src.results['nextrecord'] == 0 or src.results['returned'] == 0 or src.results['nextrecord'] > max: stop = 1 break @@ -807,13 +865,14 @@ def _harvest_csw(csw, maxrecords=10, totalrecords=float('inf')): try: _process_wms_service( known_types["WMS"], type, None, None, parent=csw) - except Exception, e: + except Exception as e: logger.error("Error registering %s:%s" % (known_types["WMS"], str(e))) elif "REST" in known_types: try: - _register_arcgis_url(ref["url"], None, None, None, parent=csw) - except Exception, e: + _register_arcgis_url( + ref["url"], None, None, None, parent=csw) + except Exception as e: logger.error("Error registering %s:%s" % (known_types["REST"], str(e))) @@ -821,7 +880,13 @@ def _harvest_csw(csw, maxrecords=10, totalrecords=float('inf')): stop = 0 -def _register_arcgis_url(url, name, username, password, owner=None, parent=None): +def _register_arcgis_url( + url, + name, + username, + password, + owner=None, + parent=None): """ Register an ArcGIS REST service URL """ @@ -832,13 +897,25 @@ def _register_arcgis_url(url, name, username, password, owner=None, parent=None) # This is a MapService try: arcserver = ArcMapService(baseurl) - except Exception, e: + except Exception as e: logger.exception(e) - if isinstance(arcserver, ArcMapService) and arcserver.spatialReference.wkid in [ - 102100, 102113, 3785, 3857, 900913]: - return_json = [_process_arcgis_service(arcserver, name, owner=owner, parent=parent)] + if isinstance( + arcserver, + ArcMapService) and arcserver.spatialReference.wkid in [ + 102100, + 102113, + 3785, + 3857, + 900913]: + return_json = [ + _process_arcgis_service( + arcserver, + name, + owner=owner, + parent=parent)] else: - return_json = [{'msg': _("Could not find any layers in a compatible projection.")}] + return_json = [ + {'msg': _("Could not find any layers in a compatible projection.")}] else: # This is a Folder @@ -878,7 +955,8 @@ def _register_arcgis_layers(service, arc=None): if existing_layer is None: try: - signals.post_save.disconnect(resourcebase_post_save, sender=Layer) + signals.post_save.disconnect( + resourcebase_post_save, sender=Layer) signals.post_save.disconnect(catalogue_post_save, sender=Layer) # Need to check if layer already exists?? @@ -980,7 +1058,12 @@ def _process_arcgis_service(arcserver, name, owner=None, parent=None): return return_dict -def _process_arcgis_folder(folder, name, services=None, owner=None, parent=None): +def _process_arcgis_folder( + folder, + name, + services=None, + owner=None, + parent=None): """ Iterate through folders and services in an ArcGIS REST service folder """ @@ -994,16 +1077,23 @@ def _process_arcgis_folder(folder, name, services=None, owner=None, parent=None) logger.debug(return_dict['msg']) else: try: - if service.spatialReference.wkid in [102100, 102113, 3785, 3857, 900913]: + if service.spatialReference.wkid in [ + 102100, 102113, 3785, 3857, 900913]: return_dict = _process_arcgis_service( service, name, owner, parent=parent) else: - return_dict['msg'] = _("Could not find any layers in a compatible projection: \ - The spatial id was: %(srs)s and the url %(url)s" % {'srs': service.spatialReference.wkid, - 'url': service.url}) + return_dict['msg'] = _( + "Could not find any layers in a compatible projection: \ + The spatial id was: %(srs)s and the url %(url)s" % { + 'srs': service.spatialReference.wkid, + 'url': service.url}) logger.debug(return_dict['msg']) except Exception as e: - logger.exception('Error uploading from the service: ' + service.url + ' ' + str(e)) + logger.exception( + 'Error uploading from the service: ' + + service.url + + ' ' + + str(e)) services.append(return_dict) @@ -1056,7 +1146,13 @@ def _register_ogp_service(url, owner=None): status=200) -def _harvest_ogp_layers(service, maxrecords=10, start=0, totalrecords=float('inf'), owner=None, institution=None): +def _harvest_ogp_layers( + service, + maxrecords=10, + start=0, + totalrecords=float('inf'), + owner=None, + institution=None): """ Query OpenGeoPortal's solr instance for layers. """ @@ -1081,7 +1177,12 @@ def _harvest_ogp_layers(service, maxrecords=10, start=0, totalrecords=float('inf while start < max: start = start + maxrecords _harvest_ogp_layers( - service, maxrecords, start, totalrecords=totalrecords, owner=owner, institution=institution) + service, + maxrecords, + start, + totalrecords=totalrecords, + owner=owner, + institution=institution) def process_ogp_results(ogp, result_json, owner=None): @@ -1091,7 +1192,7 @@ def process_ogp_results(ogp, result_json, owner=None): for doc in result_json["response"]["docs"]: try: locations = json.loads(doc["Location"]) - except: + except BaseException: continue if "tilecache" in locations: service_url = locations["tilecache"][0] @@ -1123,7 +1224,7 @@ def process_ogp_results(ogp, result_json, owner=None): service_url, service_type, None, None, parent=ogp) r_json = json.loads(response.content) service = Service.objects.get(id=r_json[0]["service_id"]) - except Exception, e: + except Exception as e: print str(e) if service: @@ -1152,21 +1253,19 @@ def process_ogp_results(ogp, result_json, owner=None): owner=None, # Assumption srid="EPSG:900913", - bbox=list(bbox), + bbox=list( + bbox), geographic_bounding_box=bbox_to_wkt( - str(bbox[0]), str(bbox[1]), + str(bbox[0]), str( + bbox[1]), str(bbox[2]), str(bbox[3]), srid="EPSG:4326") ) ) saved_layer.set_default_permissions() saved_layer.save() - service_layer, created = ServiceLayer.objects.get_or_create(service=service, - typename=typename, - defaults=dict( - title=doc[ - "LayerDisplayName"] - ) - ) + service_layer, created = ServiceLayer.objects.get_or_create( + service=service, typename=typename, defaults=dict( + title=doc["LayerDisplayName"])) if service_layer.layer is None: service_layer.layer = saved_layer service_layer.save() @@ -1202,12 +1301,12 @@ def service_detail(request, service_id): # If page is out of range (e.g. 9999), deliver last page of results. services = service_paginator.page(service_paginator.num_pages) - return render_to_response("services/service_detail.html", RequestContext(request, { - 'service': service, - 'layers': layers, - 'services': services, - 'permissions_json': _perms_info_json(service) - })) + return render_to_response("services/service_detail.html", + RequestContext(request, + {'service': service, + 'layers': layers, + 'services': services, + 'permissions_json': _perms_info_json(service)})) @login_required @@ -1230,10 +1329,10 @@ def edit_service(request, service_id): else: service_form = ServiceForm(instance=service_obj, prefix="service") - return render_to_response("services/service_edit.html", RequestContext(request, { - "service": service_obj, - "service_form": service_form - })) + return render_to_response("services/service_edit.html", + RequestContext(request, + {"service": service_obj, + "service_form": service_form})) def update_layers(service): @@ -1260,11 +1359,11 @@ def remove_service(request, service_id): service_obj = get_object_or_404(Service, pk=service_id) if not request.user.has_perm('maps.delete_service', obj=service_obj): - return HttpResponse(loader.render_to_string('401.html', - RequestContext(request, { - 'error_message': - _("You are not permitted to remove this service.") - })), status=401) + return HttpResponse( + loader.render_to_string( + '401.html', RequestContext( + request, { + 'error_message': _("You are not permitted to remove this service.")})), status=401) if request.method == 'GET': return render_to_response("services/service_remove.html", RequestContext(request, { @@ -1294,7 +1393,9 @@ def remove_service(request, service_id): @login_required def ajax_service_permissions(request, service_id): service = get_object_or_404(Service, pk=service_id) - if not request.user.has_perm("maps.change_service_permissions", obj=service): + if not request.user.has_perm( + "maps.change_service_permissions", + obj=service): return HttpResponse( 'You are not allowed to change permissions for this service', status=401, @@ -1345,7 +1446,12 @@ def create_arcgis_links(instance): ) # Create thumbnails. - bbox = urllib.pathname2url('%s,%s,%s,%s' % (instance.bbox_x0, instance.bbox_y0, instance.bbox_x1, instance.bbox_y1)) + bbox = urllib.pathname2url( + '%s,%s,%s,%s' % + (instance.bbox_x0, + instance.bbox_y0, + instance.bbox_x1, + instance.bbox_y1)) thumbnail_remote_url = instance.ows_url + 'export?LAYERS=show%3A' + str(instance.alternate) + \ '&TRANSPARENT=true&FORMAT=png&BBOX=' + bbox + '&SIZE=200%2C150&F=image&BBOXSR=4326&IMAGESR=3857' diff --git a/geonode/upload/upload.py b/geonode/upload/upload.py index 85b641e1526..778e2b3a78a 100644 --- a/geonode/upload/upload.py +++ b/geonode/upload/upload.py @@ -130,7 +130,8 @@ class UploaderSession(object): # time related info - need to store here until geoserver layer exists time_info = None - # whether the user has selected a time dimension for ImageMosaic granules or not + # whether the user has selected a time dimension for ImageMosaic granules + # or not mosaic = None append_to_mosaic_opts = None append_to_mosaic_name = None @@ -152,31 +153,48 @@ def cleanup(self): pass -def upload(name, base_file, - user=None, time_attribute=None, - time_transform_type=None, - end_time_attribute=None, end_time_transform_type=None, - presentation_strategy=None, precision_value=None, - precision_step=None, use_big_date=False, - overwrite=False, - mosaic=False, - append_to_mosaic_opts=None, append_to_mosaic_name=None, - mosaic_time_regex=None, mosaic_time_value=None, - time_presentation=None, time_presentation_res=None, - time_presentation_default_value=None, time_presentation_reference_value=None): +def upload( + name, + base_file, + user=None, + time_attribute=None, + time_transform_type=None, + end_time_attribute=None, + end_time_transform_type=None, + presentation_strategy=None, + precision_value=None, + precision_step=None, + use_big_date=False, + overwrite=False, + mosaic=False, + append_to_mosaic_opts=None, + append_to_mosaic_name=None, + mosaic_time_regex=None, + mosaic_time_value=None, + time_presentation=None, + time_presentation_res=None, + time_presentation_default_value=None, + time_presentation_reference_value=None): if user is None: user = get_default_user() if isinstance(user, basestring): user = get_user_model().objects.get(username=user) - import_session = save_step(user, name, base_file, overwrite, - mosaic=mosaic, - append_to_mosaic_opts=append_to_mosaic_opts, append_to_mosaic_name=append_to_mosaic_name, - mosaic_time_regex=mosaic_time_regex, mosaic_time_value=mosaic_time_value, - time_presentation=time_presentation, time_presentation_res=time_presentation_res, - time_presentation_default_value=time_presentation_default_value, - time_presentation_reference_value=time_presentation_reference_value) + import_session = save_step( + user, + name, + base_file, + overwrite, + mosaic=mosaic, + append_to_mosaic_opts=append_to_mosaic_opts, + append_to_mosaic_name=append_to_mosaic_name, + mosaic_time_regex=mosaic_time_regex, + mosaic_time_value=mosaic_time_value, + time_presentation=time_presentation, + time_presentation_res=time_presentation_res, + time_presentation_default_value=time_presentation_default_value, + time_presentation_reference_value=time_presentation_reference_value) upload_session = UploaderSession( base_file=base_file, @@ -208,12 +226,20 @@ def _log(msg, *args): logger.info(msg, *args) -def save_step(user, layer, spatial_files, overwrite=True, - mosaic=False, - append_to_mosaic_opts=None, append_to_mosaic_name=None, - mosaic_time_regex=None, mosaic_time_value=None, - time_presentation=None, time_presentation_res=None, - time_presentation_default_value=None, time_presentation_reference_value=None): +def save_step( + user, + layer, + spatial_files, + overwrite=True, + mosaic=False, + append_to_mosaic_opts=None, + append_to_mosaic_name=None, + mosaic_time_regex=None, + mosaic_time_value=None, + time_presentation=None, + time_presentation_res=None, + time_presentation_default_value=None, + time_presentation_reference_value=None): _log('Uploading layer: [%s], files [%s]', layer, spatial_files) if len(spatial_files) > 1: @@ -282,12 +308,14 @@ def save_step(user, layer, spatial_files, overwrite=True, try: # importer tracks ids by autoincrement but is prone to corruption # which potentially may reset the id - hopefully prevent this... - upload_next_id = Upload.objects.all().aggregate(Max('import_id')).values()[0] + upload_next_id = Upload.objects.all().aggregate( + Max('import_id')).values()[0] upload_next_id = upload_next_id if upload_next_id else 0 # next_id = next_id + 1 if next_id else 1 importer_sessions = gs_uploader.get_sessions() - last_importer_session = importer_sessions[len(importer_sessions)-1].id if importer_sessions else 0 + last_importer_session = importer_sessions[len( + importer_sessions) - 1].id if importer_sessions else 0 next_id = max(int(last_importer_session), int(upload_next_id)) + 1 next_id = max(int(last_importer_session), int(upload_next_id)) + 1 @@ -309,10 +337,16 @@ def save_step(user, layer, spatial_files, overwrite=True, # if mosaic_time_regex and mosaic_time_value: if mosaic: # we want to ingest as ImageMosaic - target_store = import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_mosaic_name, - mosaic_time_regex, mosaic_time_value, time_presentation, - time_presentation_res, time_presentation_default_value, - time_presentation_reference_value) + target_store = import_imagemosaic_granules( + spatial_files, + append_to_mosaic_opts, + append_to_mosaic_name, + mosaic_time_regex, + mosaic_time_value, + time_presentation, + time_presentation_res, + time_presentation_default_value, + time_presentation_reference_value) # moving forward with a regular Importer session import_session = gs_uploader.upload_files( @@ -474,7 +508,10 @@ def build_att_remap_transform(att): 'field': att, 'target': 'org.geotools.data.postgis.PostGISDialect$XDate'} - use_big_date = getattr(settings, 'USE_BIG_DATE', False) and not upload_session.geogig + use_big_date = getattr( + settings, + 'USE_BIG_DATE', + False) and not upload_session.geogig if time_attribute: if time_transform_type: @@ -592,7 +629,9 @@ def final_step(upload_session, user): import_session.commit() if not publishing: - raise LayerNotReady("Expected to find layer named '%s' in geoserver" % name) + raise LayerNotReady( + "Expected to find layer named '%s' in geoserver" % + name) _log('Creating style for [%s]', name) # get_files will not find the sld if it doesn't match the base name @@ -626,13 +665,13 @@ def final_step(upload_session, user): if style is None: try: style = cat.get_style(name) - except: + except BaseException: logger.warn('Could not retreive the Layer default Style name') # what are we doing with this var? msg = 'No style could be created for the layer, falling back to POINT default one' try: style = cat.get_style(name + '_layer') - except: + except BaseException: style = cat.get_style('point') logger.warn(msg) e.args = (msg,) @@ -695,19 +734,26 @@ def final_step(upload_session, user): else: # saved_layer = Layer.objects.filter(name=upload_session.append_to_mosaic_name) # created = False - saved_layer, created = Layer.objects.get_or_create(name=upload_session.append_to_mosaic_name) + saved_layer, created = Layer.objects.get_or_create( + name=upload_session.append_to_mosaic_name) try: if saved_layer.temporal_extent_start and end: - if pytz.utc.localize(saved_layer.temporal_extent_start, is_dst=False) < end: + if pytz.utc.localize( + saved_layer.temporal_extent_start, + is_dst=False) < end: saved_layer.temporal_extent_end = end - Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( + Layer.objects.filter( + name=upload_session.append_to_mosaic_name).update( temporal_extent_end=end) else: saved_layer.temporal_extent_start = end - Layer.objects.filter(name=upload_session.append_to_mosaic_name).update( + Layer.objects.filter( + name=upload_session.append_to_mosaic_name).update( temporal_extent_start=end) except Exception as e: - _log('There was an error updating the mosaic temporal extent: ' + str(e)) + _log( + 'There was an error updating the mosaic temporal extent: ' + + str(e)) else: saved_layer, created = Layer.objects.get_or_create( name=task.layer.name, @@ -749,7 +795,8 @@ def final_step(upload_session, user): zf.extract(xml_file[0], os.path.dirname(archive)) # Assign the absolute path to this file xml_file[0] = os.path.dirname(archive) + '/' + xml_file[0] - identifier, vals, regions, keywords = set_metadata(open(xml_file[0]).read()) + identifier, vals, regions, keywords = set_metadata( + open(xml_file[0]).read()) regions_resolved, regions_unresolved = resolve_regions(regions) keywords.extend(regions_unresolved) @@ -798,10 +845,16 @@ def final_step(upload_session, user): return saved_layer -def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_mosaic_name, - mosaic_time_regex, mosaic_time_value, time_presentation, - time_presentation_res, time_presentation_default_value, - time_presentation_reference_value): +def import_imagemosaic_granules( + spatial_files, + append_to_mosaic_opts, + append_to_mosaic_name, + mosaic_time_regex, + mosaic_time_value, + time_presentation, + time_presentation_res, + time_presentation_default_value, + time_presentation_reference_value): # The very first step is to rename the granule by adding the selected regex # matching value to the filename. @@ -811,7 +864,14 @@ def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_ basename = os.path.basename(f) head, tail = os.path.splitext(basename) - dst_file = os.path.join(dirname, head.replace("_", "-") + "_" + mosaic_time_value + tail) + dst_file = os.path.join( + dirname, + head.replace( + "_", + "-") + + "_" + + mosaic_time_value + + tail) os.rename(f, dst_file) spatial_files[0].base_file = dst_file @@ -828,8 +888,8 @@ def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_ context = { "abs_path_flag": "True", - "time_attr": "time", - "aux_metadata_flag": "False", + "time_attr": "time", + "aux_metadata_flag": "False", "mosaic_time_regex": mosaic_time_regex, "db_host": db['HOST'], "db_port": db['PORT'], @@ -886,9 +946,14 @@ def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_ z.write(dst_file, arcname=head + "_" + mosaic_time_value + tail) z.write(dirname + '/indexer.properties', arcname='indexer.properties') - z.write(dirname + '/datastore.properties', arcname='datastore.properties') + z.write( + dirname + + '/datastore.properties', + arcname='datastore.properties') if mosaic_time_regex: - z.write(dirname + '/timeregex.properties', arcname='timeregex.properties') + z.write( + dirname + '/timeregex.properties', + arcname='timeregex.properties') z.close() @@ -904,8 +969,13 @@ def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_ # configure time as LIST if mosaic_time_regex: - set_time_dimension(cat, name, time_presentation, time_presentation_res, - time_presentation_default_value, time_presentation_reference_value) + set_time_dimension( + cat, + name, + time_presentation, + time_presentation_res, + time_presentation_default_value, + time_presentation_reference_value) # - since GeoNode will uploade the first granule again through the Importer, we need to / # delete the one created by the gs_config diff --git a/geonode/upload/views.py b/geonode/upload/views.py index 2d0e1993ea3..88fe50014ed 100644 --- a/geonode/upload/views.py +++ b/geonode/upload/views.py @@ -126,7 +126,8 @@ def __init__(self, if json_opts is None: json_opts = {} content = json.dumps(obj, **json_opts) - super(JSONResponse, self).__init__(content, content_type, *args, **kwargs) + super(JSONResponse, self).__init__( + content, content_type, *args, **kwargs) def _error_response(req, exception=None, errors=None, force_ajax=True): @@ -229,8 +230,13 @@ def _next_step_response(req, upload_session, force_ajax=True): force_ajax=force_ajax) if req.is_ajax() or force_ajax: content_type = 'text/html' if not req.is_ajax() else None - return json_response(redirect_to=reverse('data_upload', args=[next]) + "?id=%s" % req.GET['id'], - content_type=content_type) + return json_response( + redirect_to=reverse( + 'data_upload', + args=[next]) + + "?id=%s" % + req.GET['id'], + content_type=content_type) # return HttpResponseRedirect(reverse('data_upload', args=[next])) @@ -334,7 +340,8 @@ def data_upload_progress(req): if 'id' in req.GET: upload_id = str(req.GET['id']) if upload_id in req.session: - upload_obj = get_object_or_404(Upload, import_id=upload_id, user=req.user) + upload_obj = get_object_or_404( + Upload, import_id=upload_id, user=req.user) upload_session = upload_obj.get_session() else: upload_session = req.session[upload_id] @@ -552,25 +559,27 @@ def final_step_view(req, upload_session): # this response is different then all of the other views in the # upload as it does not return a response as a json object + _json_response = None try: - json_response = json_response( + _json_response = json_response( { - 'url': saved_layer.get_absolute_url(), - 'bbox': saved_layer.bbox_string, - 'crs': { - 'type': 'name', - 'properties': saved_layer.srid - }, - 'success': True + 'url': saved_layer.get_absolute_url(), + 'bbox': saved_layer.bbox_string, + 'crs': { + 'type': 'name', + 'properties': saved_layer.srid + }, + 'success': True } ) - except: - json_response = json_response( + except BaseException: + _json_response = json_response( {'url': saved_layer.get_absolute_url(), 'success': True - } + } ) - return json_response + return _json_response + _steps = { 'save': save_step_view, @@ -686,7 +695,8 @@ def view(req, step): req, {})) - upload_obj = get_object_or_404(Upload, import_id=upload_id, user=req.user) + upload_obj = get_object_or_404( + Upload, import_id=upload_id, user=req.user) session = upload_obj.get_session() if session: upload_session = session @@ -710,7 +720,7 @@ def view(req, step): try: resp_js = json.loads(resp.content) delete_session = resp_js.get('status') != 'pending' - except: + except BaseException: pass if delete_session: @@ -807,7 +817,8 @@ def delete(self, request, *args, **kwargs): self.object = self.get_object() self.object.delete() if request.is_ajax(): - response = JSONResponse(True, {}, response_content_type(self.request)) + response = JSONResponse( + True, {}, response_content_type(self.request)) response['Content-Disposition'] = 'inline; filename=files.json' return response else: diff --git a/geonode/utils.py b/geonode/utils.py index 4e77a9fd173..b2ae6f1e994 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -292,12 +292,13 @@ def layer_config(l, user=None): for source in sources.values() if 'url' in source] if 'geonode.geoserver' in settings.INSTALLED_APPS: - if len(sources.keys()) > 0 and not settings.MAP_BASELAYERS[0]['source']['url'] in source_urls: + if len(sources.keys( + )) > 0 and not settings.MAP_BASELAYERS[0]['source']['url'] in source_urls: keys = sorted(sources.keys()) settings.MAP_BASELAYERS[0]['source'][ 'title'] = 'Local Geoserver' - sources[ - str(int(keys[-1]) + 1)] = settings.MAP_BASELAYERS[0]['source'] + sources[str(int(keys[-1]) + 1) + ] = settings.MAP_BASELAYERS[0]['source'] def _base_source(source): base_source = copy.deepcopy(source) @@ -312,8 +313,8 @@ def _base_source(source): _base_source, sources.values()): if len(sources.keys()) > 0: - sources[ - str(int(max(sources.keys(), key=int)) + 1)] = lyr["source"] + sources[str(int(max(sources.keys(), key=int)) + 1) + ] = lyr["source"] # adding remote services sources from geonode.services.models import Service @@ -414,8 +415,12 @@ def source_config(self, access_token): request_params['access_token'] = [access_token] encoded_params = urllib.urlencode(request_params, doseq=True) - parsed_url = urlparse.SplitResult(my_url.scheme, my_url.netloc, my_url.path, - encoded_params, my_url.fragment) + parsed_url = urlparse.SplitResult( + my_url.scheme, + my_url.netloc, + my_url.path, + encoded_params, + my_url.fragment) cfg["url"] = parsed_url.geturl() else: cfg["url"] = self.ows_url @@ -514,7 +519,8 @@ def _baselayer(lyr, order): u = uuid.uuid1() access_token = u.hex - DEFAULT_MAP_CONFIG = _default_map.viewer_json(user, access_token, *DEFAULT_BASE_LAYERS) + DEFAULT_MAP_CONFIG = _default_map.viewer_json( + user, access_token, *DEFAULT_BASE_LAYERS) return DEFAULT_MAP_CONFIG, DEFAULT_BASE_LAYERS @@ -558,7 +564,8 @@ def resolve_object(request, model, query, permission='base.view_resourcebase', raise Http404 allowed = True - if permission.split('.')[-1] in ['change_layer_data', 'change_layer_style']: + if permission.split('.')[-1] in ['change_layer_data', + 'change_layer_style']: if obj.__class__.__name__ == 'Layer': obj_to_check = obj if permission: @@ -654,7 +661,8 @@ def format_urls(a, values): def build_abstract(resourcebase, url=None, includeURL=True): if resourcebase.abstract and url and includeURL: - return u"{abstract} -- [{url}]({url})".format(abstract=resourcebase.abstract, url=url) + return u"{abstract} -- [{url}]({url})".format( + abstract=resourcebase.abstract, url=url) else: return resourcebase.abstract @@ -679,8 +687,10 @@ def build_social_links(request, resourcebase): host=request.get_host(), path=request.get_full_path()) # Don't use datetime strftime() because it requires year >= 1900 - # see https://docs.python.org/2/library/datetime.html#strftime-strptime-behavior - date = '{0.month:02d}/{0.day:02d}/{0.year:4d}'.format(resourcebase.date) if resourcebase.date else None + # see + # https://docs.python.org/2/library/datetime.html#strftime-strptime-behavior + date = '{0.month:02d}/{0.day:02d}/{0.year:4d}'.format( + resourcebase.date) if resourcebase.date else None abstract = build_abstract(resourcebase, url=social_url, includeURL=True) caveats = build_caveats(resourcebase) hashtags = ",".join(getattr(settings, 'TWITTER_HASHTAGS', [])) @@ -733,7 +743,9 @@ def check_shp_columnnames(layer): list_col_original.append(field_name) try: for i in range(0, inLayerDefn.GetFieldCount()): - field_name = unicode(inLayerDefn.GetFieldDefn(i).GetName(), layer.charset) + field_name = unicode( + inLayerDefn.GetFieldDefn(i).GetName(), + layer.charset) if not a.match(field_name): new_field_name = custom_slugify(field_name) @@ -756,12 +768,17 @@ def check_shp_columnnames(layer): return True, None, None else: for key in list_col.keys(): - qry = u"ALTER TABLE {0} RENAME COLUMN \"{1}\" TO \"{2}\"".format(inLayer.GetName(), key, list_col[key]) + qry = u"ALTER TABLE {0} RENAME COLUMN \"{1}\" TO \"{2}\"".format( + inLayer.GetName(), key, list_col[key]) inDataSource.ExecuteSQL(qry.encode(layer.charset)) return True, None, list_col -def set_attributes(layer, attribute_map, overwrite=False, attribute_stats=None): +def set_attributes( + layer, + attribute_map, + overwrite=False, + attribute_stats=None): """ *layer*: a geonode.layers.models.Layer instance *attribute_map*: a list of 2-lists specifying attribute names and types, example: [ ['id', 'Integer'], ... ] @@ -795,7 +812,8 @@ def set_attributes(layer, attribute_map, overwrite=False, attribute_stats=None): # store description and attribute_label in attribute_map attribute[attribute_map_dict['description']] = la.description attribute[attribute_map_dict['label']] = la.attribute_label - attribute[attribute_map_dict['display_order']] = la.display_order + attribute[attribute_map_dict['display_order'] + ] = la.display_order if overwrite or not lafound: logger.debug( "Going to delete [%s] for [%s]", @@ -868,7 +886,8 @@ def designals(): for signalname in signalnames: signaltype = getattr(models.signals, signalname) - logger.debug("RETRIEVE: %s: %d" % (signalname, len(signaltype.receivers))) + logger.debug("RETRIEVE: %s: %d" % + (signalname, len(signaltype.receivers))) signals_store[signalname] = [] signals = signaltype.receivers[:] for signal in signals: @@ -888,7 +907,7 @@ def designals(): # - case id(function) or uid try: receiv_call = id_to_obj(lookup[0]) - except: + except BaseException: uid = lookup[0] if isinstance(lookup[1], tuple): @@ -909,9 +928,13 @@ def designals(): 'uid': uid, 'is_weak': is_weak, 'sender_ista': sender_ista, 'sender_call': sender_call, 'receiv_call': receiv_call, - }) + }) - signaltype.disconnect(receiver=receiv_call, sender=sender_ista, weak=is_weak, dispatch_uid=uid) + signaltype.disconnect( + receiver=receiv_call, + sender=sender_ista, + weak=is_weak, + dispatch_uid=uid) def resignals(): @@ -921,12 +944,19 @@ def resignals(): signals = signals_store[signalname] signaltype = getattr(models.signals, signalname) for signal in signals: - signaltype.connect(signal['receiv_call'], sender=signal['sender_ista'], - weak=signal['is_weak'], dispatch_uid=signal['uid']) + signaltype.connect( + signal['receiv_call'], + sender=signal['sender_ista'], + weak=signal['is_weak'], + dispatch_uid=signal['uid']) def run_subprocess(*cmd, **kwargs): - p = subprocess.Popen(' '.join(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) + p = subprocess.Popen( + ' '.join(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs) stdout = StringIO() stderr = StringIO() buff_size = 1024 diff --git a/pavement.py b/pavement.py index d0cea7daa9c..70c8808ef04 100644 --- a/pavement.py +++ b/pavement.py @@ -39,7 +39,7 @@ try: from geonode.settings import GEONODE_APPS -except: +except BaseException: # probably trying to run install_win_deps. pass @@ -89,11 +89,23 @@ def setup_geoserver(options): geoserver_dir = path('geoserver') - geoserver_bin = download_dir / os.path.basename(dev_config['GEOSERVER_URL']) - jetty_runner = download_dir / os.path.basename(dev_config['JETTY_RUNNER_URL']) - - grab(options.get('geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary") - grab(options.get('jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner") + geoserver_bin = download_dir / \ + os.path.basename(dev_config['GEOSERVER_URL']) + jetty_runner = download_dir / \ + os.path.basename(dev_config['JETTY_RUNNER_URL']) + + grab( + options.get( + 'geoserver', + dev_config['GEOSERVER_URL']), + geoserver_bin, + "geoserver binary") + grab( + options.get( + 'jetty', + dev_config['JETTY_RUNNER_URL']), + jetty_runner, + "jetty runner") if not geoserver_dir.exists(): geoserver_dir.makedirs() @@ -117,7 +129,8 @@ def _install_data_dir(): original_data_dir = path('geoserver/geoserver/data') justcopy(original_data_dir, target_data_dir) - config = path('geoserver/data/security/auth/geonodeAuthProvider/config.xml') + config = path( + 'geoserver/data/security/auth/geonodeAuthProvider/config.xml') with open(config) as f: xml = f.read() m = re.search('baseUrl>([^<]+)', xml) @@ -178,7 +191,7 @@ def win_install_deps(options): grab_winfiles(url, tempfile, package) try: easy_install.main([tempfile]) - except Exception, e: + except Exception as e: failed = True print "install failed with error: ", e os.remove(tempfile) @@ -320,7 +333,8 @@ def stop(): """ Stop GeoNode """ - # windows needs to stop the geoserver first b/c we can't tell which python is running, so we kill everything + # windows needs to stop the geoserver first b/c we can't tell which python + # is running, so we kill everything stop_geoserver() info("Stopping GeoNode ...") stop_django() @@ -360,7 +374,8 @@ def start_geoserver(options): sys.exit(1) download_dir = path('downloaded').abspath() - jetty_runner = download_dir / os.path.basename(dev_config['JETTY_RUNNER_URL']) + jetty_runner = download_dir / \ + os.path.basename(dev_config['JETTY_RUNNER_URL']) data_dir = path('geoserver/data').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() @@ -372,11 +387,12 @@ def start_geoserver(options): javapath = "java" loggernullpath = os.devnull - # checking if our loggernullpath exists and if not, reset it to something manageable + # checking if our loggernullpath exists and if not, reset it to + # something manageable if loggernullpath == "nul": try: open("../../downloaded/null.txt", 'w+').close() - except IOError, e: + except IOError as e: print "Chances are that you have Geoserver currently running. You \ can either stop all servers with paver stop or start only \ the django application with paver start_django." @@ -385,12 +401,13 @@ def start_geoserver(options): try: sh(('java -version')) - except: + except BaseException: print "Java was not found in your path. Trying some other options: " javapath_opt = None if os.environ.get('JAVA_HOME', None): print "Using the JAVA_HOME environment variable" - javapath_opt = os.path.join(os.path.abspath(os.environ['JAVA_HOME']), "bin", "java.exe") + javapath_opt = os.path.join(os.path.abspath( + os.environ['JAVA_HOME']), "bin", "java.exe") elif options.get('java_path'): javapath_opt = options.get('java_path') else: @@ -445,7 +462,7 @@ def test_javascript(options): @task @cmdopts([ ('name=', 'n', 'Run specific tests.') - ]) +]) def test_integration(options): """ Run GeoNode's Integration test suite against the external apps @@ -465,8 +482,8 @@ def test_integration(options): sh('sleep 30') call_task('setup_data') sh(('python manage.py test %s' - ' --noinput --liveserver=localhost:8000' % name)) - except BuildFailure, e: + ' --noinput --liveserver=localhost:8000' % name)) + except BuildFailure as e: info('Tests failed! %s' % str(e)) else: success = True @@ -594,13 +611,13 @@ def deb(options): sh('debuild -uc -us -A') elif key is None and ppa is not None: # A sources package, signed by daemon - sh('debuild -S') + sh('debuild -S') elif key is not None and ppa is None: - # A signed installable package - sh('debuild -k%s -A' % key) + # A signed installable package + sh('debuild -k%s -A' % key) elif key is not None and ppa is not None: - # A signed, source package - sh('debuild -k%s -S' % key) + # A signed, source package + sh('debuild -k%s -S' % key) if ppa is not None: sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version)) @@ -677,7 +694,9 @@ def kill(arg1, arg2): running = False for line in lines: # this kills all java.exe and python including self in windows - if ('%s' % arg2 in line) or (os.name == 'nt' and '%s' % arg1 in line): + if ('%s' % + arg2 in line) or (os.name == 'nt' and '%s' % + arg1 in line): running = True # Get pid diff --git a/setup.cfg b/setup.cfg index f74064feeb0..1c6bcb9b976 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,4 @@ [flake8] max-line-length = 120 -exclude=geonode/*/migrations/* +exclude=geonode/*/migrations/*,management,scripts,docs,static,migrations +ignore=E121,E122,E124,E126,E226