Merge pull request #1559 from DemocracyClub/feature/timestamps

Feature/timestamps

README.md

@@ -33,3 +33,9 @@ This codebase was originally forked from
 [mysociety/yournextrepresentative](http://github.com/mysociety/yournextrepresentative)
 We no longer track the upstream but we thank [mySociety](http://mysociety.org/)
 for their work on the project which we have been able to build on.
+
+# API Versions
+
+v0.9 is legacy code and is now frozen. v1.0 is currently in alpha. We plan on publishing a v1 API once we have some more feedback from users and we think it’s stable enough.
+
+

ynr/apps/api/next/filters.py

@@ -0,0 +1,10 @@
+from django_filters import filterset, filters
+
+
+class LastUpdatedMixin(filterset.FilterSet):
+    last_updated = filters.IsoDateTimeFilter(
+        field_name="modified",
+        lookup_expr="gt",
+        label="Last updated",
+        help_text="An ISO datetime",
+    )

ynr/apps/api/next/serializers.py

@@ -23,10 +23,11 @@
 class OrganizationSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = popolo_models.Organization
-        fields = ("url", "name", "slug")
+        fields = ("url", "name", "slug", "last_updated", "created")
 
     url = serializers.HyperlinkedIdentityField(
         view_name="organization-detail",
         lookup_field="slug",
         lookup_url_kwarg="slug",
     )
+    last_updated = serializers.DateTimeField(source="modified")

ynr/apps/api/next/views.py

@@ -3,10 +3,11 @@
 from rest_framework import pagination, viewsets
 
 import candidates.api.next.serializers
-from api.next import serializers
+import api.next.serializers
 from candidates import models as extra_models
-from candidates.filters import LoggedActionAPIFilter
 from popolo.models import Organization
+from candidates.filters import LoggedActionAPIFilter
+from popolo.api.next.filters import OrganizationFilter
 
 
 def parse_date(date_text):
@@ -33,8 +34,9 @@ class OrganizationViewSet(viewsets.ReadOnlyModelViewSet):
         .exclude(classification="Party")
     )
     lookup_field = "slug"
-    serializer_class = serializers.OrganizationSerializer
+    serializer_class = api.next.serializers.OrganizationSerializer
     pagination_class = ResultsSetPagination
+    filterset_class = OrganizationFilter
 
 
 class LoggedActionViewSet(viewsets.ReadOnlyModelViewSet):

ynr/apps/api/templates/api/next_home.html

@@ -1,5 +1,5 @@
 {% extends "api/api-base.html" %}
-{% load staticfiles %}
+{% load static %}
 {% load api_docs %}
 {% load markdown_deux_tags %}
 {% block api_page_title %}

ynr/apps/api/v09/serializers.py

@@ -1,3 +1,4 @@
+# v0.9 is legacy code
 from rest_framework import serializers
 from rest_framework.fields import JSONField
 from sorl_thumbnail_serializer.fields import HyperlinkedSorlImageField

ynr/apps/api/v09/views.py

@@ -1,3 +1,4 @@
+# v0.9 is legacy code
 import json
 import subprocess
 import sys
@@ -222,7 +223,7 @@ def get_queryset(self):
         if date_qs:
             date = parser.parse(date_qs)
             queryset = queryset.filter(
-                Q(updated_at__gte=date) | Q(memberships__updated_at__gte=date)
+                Q(modified__gte=date) | Q(memberships__modified__gte=date)
             )
         return queryset
 

ynr/apps/candidates/migrations/0070_auto_20210617_1506.py

@@ -0,0 +1,37 @@
+# Generated by Django 2.2.20 on 2021-06-17 14:06
+
+from django.db import migrations
+import django.utils.timezone
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("candidates", "0069_ballot_voting_system")]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="ballot",
+            options={
+                "get_latest_by": "modified",
+                "ordering": ("-modified", "-created"),
+            },
+        ),
+        migrations.AddField(
+            model_name="ballot",
+            name="created",
+            field=django_extensions.db.fields.CreationDateTimeField(
+                auto_now_add=True,
+                default=django.utils.timezone.now,
+                verbose_name="created",
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="ballot",
+            name="modified",
+            field=django_extensions.db.fields.ModificationDateTimeField(
+                auto_now=True, verbose_name="modified"
+            ),
+        ),
+    ]

ynr/apps/candidates/migrations/0071_add_created_data.py

@@ -0,0 +1,25 @@
+# Generated by Django 2.2.20 on 2021-06-17 14:08
+
+from django.db import migrations
+
+from elections.helpers import four_weeks_before_election_date
+
+
+def add_created_date(apps, schema_editor):
+    Ballot = apps.get_model("candidates", "Ballot")
+    for ballot in Ballot.objects.select_related("election").iterator():
+        ballot.created = four_weeks_before_election_date(
+            election=ballot.election
+        )
+        ballot.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("candidates", "0070_auto_20210617_1506")]
+
+    operations = [
+        migrations.RunPython(
+            code=add_created_date, reverse_code=migrations.RunPython.noop
+        )
+    ]

ynr/apps/candidates/models/popolo_extra.py

@@ -8,6 +8,7 @@
 from django.utils import timezone
 from django.utils.html import mark_safe
 from django.utils.functional import cached_property
+from django_extensions.db.models import TimeStampedModel
 
 from candidates.models.auth import TRUSTED_TO_LOCK_GROUP_NAME
 from elections.models import Election
@@ -148,7 +149,7 @@ def no_results(self):
         ).distinct()
 
 
-class Ballot(models.Model):
+class Ballot(TimeStampedModel, models.Model):
 
     VOTING_SYSTEM_FPTP = "FPTP"
 

ynr/apps/elections/helpers.py

@@ -1,4 +1,5 @@
 from functools import update_wrapper
+from django.utils import timezone
 
 from candidates.models import Ballot
 
@@ -33,3 +34,19 @@ def __call__(self, request, *args, **kwargs):
 
         self.__name__ = self.__qualname__ = view.__name__
         return view(request, *args, **kwargs)
+
+
+def four_weeks_before_election_date(election):
+    """
+    Takes an election object and a datetime object four weeks prior to the
+    election date.
+    Used in data migrations where we want to set a realistic created timestamp
+    on old objects.
+    """
+    election_date = election.election_date - timezone.timedelta(weeks=4)
+    return timezone.datetime(
+        election_date.year,
+        election_date.month,
+        election_date.day,
+        tzinfo=timezone.utc,
+    )

ynr/apps/elections/tests/test_helpers.py

@@ -0,0 +1,12 @@
+from django.utils import timezone
+
+from elections.models import Election
+from elections import helpers
+
+
+class TestHelpers:
+    def test_four_weeks_before_election_date(self):
+        election = Election(election_date=timezone.datetime(2021, 5, 6).date())
+        expected = timezone.datetime(2021, 4, 8, 0, 0, 0, tzinfo=timezone.utc)
+        result = helpers.four_weeks_before_election_date(election=election)
+        assert result == expected

ynr/apps/parties/api/next/api_views.py

@@ -5,6 +5,7 @@
 from django.views.decorators.cache import cache_page
 from django.views import View
 from rest_framework import viewsets
+from parties.api.next.filters import PartyFilter
 
 from api.helpers import DefaultPageNumberPagination
 
@@ -25,6 +26,7 @@ class PartyViewSet(viewsets.ReadOnlyModelViewSet):
     serializer_class = PartySerializer
     lookup_field = "ec_id"
     pagination_class = DefaultPageNumberPagination
+    filterset_class = PartyFilter
 
     def retrieve(self, request, *args, **kwargs):
         """

ynr/apps/parties/api/next/filters.py

@@ -0,0 +1,8 @@
+from parties.models import Party
+from api.next.filters import LastUpdatedMixin
+
+
+class PartyFilter(LastUpdatedMixin):
+    class Meta:
+        model = Party
+        fields = ["last_updated"]

ynr/apps/parties/api/next/serializers.py

@@ -51,6 +51,8 @@ class Meta:
             "emblems",
             "descriptions",
             "legacy_slug",
+            "created",
+            "last_updated",
         )
         swagger_schema_fields = {"description": model.__doc__}
 
@@ -60,6 +62,7 @@ class Meta:
     default_emblem = DefaultPartyEmblemSerializer(read_only=True)
     emblems = PartyEmblemSerializer(many=True, read_only=True)
     descriptions = PartyDescriptionSerializer(many=True, read_only=True)
+    last_updated = serializers.DateTimeField(source="modified")
 
 
 class PartyRegisterSerializer(serializers.Serializer):

ynr/apps/people/api/next/api_views.py

@@ -10,6 +10,7 @@
 
 import people.api.next.serializers
 from api.next.views import ResultsSetPagination
+from people.api.next.filters import PersonFilter
 from candidates import models as extra_models
 from candidates.api.next.serializers import LoggedActionSerializer
 from people.models import Person, PersonImage
@@ -39,7 +40,7 @@ def get_queryset(self):
         if date_qs:
             date = parser.parse(date_qs)
             queryset = queryset.filter(
-                Q(updated_at__gte=date) | Q(memberships__updated_at__gte=date)
+                Q(modified__gte=date) | Q(memberships__modified__gte=date)
             )
         return queryset
 
@@ -88,6 +89,7 @@ def versions(self, request, pk=None, **kwargs):
 
     serializer_class = people.api.next.serializers.PersonSerializer
     pagination_class = ResultsSetPagination
+    filterset_class = PersonFilter
 
 
 class PersonRedirectViewSet(viewsets.ReadOnlyModelViewSet):

ynr/apps/people/api/next/filters.py

@@ -0,0 +1,8 @@
+from people.models import Person
+from api.next.filters import LastUpdatedMixin
+
+
+class PersonFilter(LastUpdatedMixin):
+    class Meta:
+        model = Person
+        fields = ["last_updated"]

ynr/apps/people/api/next/serializers.py

@@ -54,6 +54,8 @@ class Meta:
         model = popolo_models.ContactDetail
         fields = ("contact_type", "label", "note", "value")
 
+        last_updated = serializers.DateTimeField(source="modified")
+
 
 class SourceSerializer(serializers.ModelSerializer):
     class Meta:
@@ -83,6 +85,7 @@ class Meta:
             "url",
             "versions_url",
             "history_url",
+            "created",
             "last_updated",
             "honorific_prefix",
             "name",
@@ -107,7 +110,7 @@ class Meta:
     history_url = serializers.HyperlinkedIdentityField(
         view_name="person-history"
     )
-    last_updated = serializers.DateTimeField(source="updated_at")
+    last_updated = serializers.DateTimeField(source="modified")
     identifiers = PersonIdentifierSerializer(
         many=True, read_only=True, source="tmp_person_identifiers"
     )

ynr/apps/people/merging.py

@@ -93,8 +93,8 @@ class PersonMerger:
             ("duplicate_suggestion_other_person", "merge_duplicate_suggestion"),
             # Discarded
             ("id", "discard_data"),
-            ("created_at", "discard_data"),
-            ("updated_at", "discard_data"),
+            ("created", "discard_data"),
+            ("modified", "discard_data"),
             ("edit_limitations", "discard_data"),
             ("sources", "discard_data"),
         )

ynr/apps/people/migrations/0030_auto_20210616_1642.py

@@ -0,0 +1,30 @@
+# Generated by Django 2.2.20 on 2021-06-16 15:42
+
+from django.db import migrations
+import django.utils.timezone
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("people", "0029_default_versions_to_list")]
+
+    operations = [
+        migrations.AddField(
+            model_name="person",
+            name="created",
+            field=django_extensions.db.fields.CreationDateTimeField(
+                auto_now_add=True,
+                default=django.utils.timezone.now,
+                verbose_name="created",
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="person",
+            name="modified",
+            field=django_extensions.db.fields.ModificationDateTimeField(
+                auto_now=True, verbose_name="modified"
+            ),
+        ),
+    ]

ynr/apps/people/migrations/0031_copy_created.py

@@ -0,0 +1,26 @@
+# Generated by Django 2.2.20 on 2021-06-16 15:59
+
+from django.db import migrations
+
+
+def copy_created_at(apps, schema_editor):
+    Person = apps.get_model("people", "Person")
+    for person in Person.objects.iterator():
+        person.created = person.created_at
+        person.save()
+
+
+def copy_created(apps, schema_editor):
+    Person = apps.get_model("people", "Person")
+    for person in Person.objects.iterator():
+        person.created_at = person.created
+        person.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("people", "0030_auto_20210616_1642")]
+
+    operations = [
+        migrations.RunPython(code=copy_created_at, reverse_code=copy_created)
+    ]

ynr/apps/people/migrations/0032_auto_20210616_1700.py

@@ -0,0 +1,13 @@
+# Generated by Django 2.2.20 on 2021-06-16 16:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("people", "0031_copy_created")]
+
+    operations = [
+        migrations.RemoveField(model_name="person", name="created_at"),
+        migrations.RemoveField(model_name="person", name="updated_at"),
+    ]