diff --git a/archivebox.egg-info/requires.txt b/archivebox.egg-info/requires.txt index ca279875..71dc253d 100644 --- a/archivebox.egg-info/requires.txt +++ b/archivebox.egg-info/requires.txt @@ -4,7 +4,6 @@ mypy-extensions==0.4.3 base32-crockford==0.3.0 django==3.0.8 django-extensions==3.0.3 -django-taggit==1.3.0 dateparser ipython youtube-dl diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py index b28d6e52..55c68e16 100644 --- a/archivebox/core/admin.py +++ b/archivebox/core/admin.py @@ -9,9 +9,10 @@ from django.utils.html import format_html from django.utils.safestring import mark_safe from django.shortcuts import render, redirect from django.contrib.auth import get_user_model +from django import forms from core.models import Snapshot -from core.forms import AddLinkForm +from core.forms import AddLinkForm, TagField from core.utils import get_icons from util import htmldecode, urldecode, ansi_to_html @@ -55,6 +56,32 @@ def delete_snapshots(modeladmin, request, queryset): delete_snapshots.short_description = "Delete" +class SnapshotAdminForm(forms.ModelForm): + tags = TagField(required=False) + + class Meta: + model = Snapshot + fields = "__all__" + + def save(self, commit=True): + # Based on: https://stackoverflow.com/a/49933068/3509554 + + # Get the unsave instance + instance = forms.ModelForm.save(self, False) + tags = self.cleaned_data.pop("tags") + + #update save_m2m + def new_save_m2m(): + instance.save_tags(tags) + + # Do we need to save all changes now? + self.save_m2m = new_save_m2m + if commit: + instance.save() + + return instance + + class SnapshotAdmin(admin.ModelAdmin): list_display = ('added', 'title_str', 'url_str', 'files', 'size') sort_fields = ('title_str', 'url_str', 'added') @@ -65,6 +92,7 @@ class SnapshotAdmin(admin.ModelAdmin): ordering = ['-added'] actions = [delete_snapshots, overwrite_snapshots, update_snapshots, update_titles, verify_snapshots] actions_template = 'admin/actions_as_select.html' + form = SnapshotAdminForm def get_queryset(self, request): return super().get_queryset(request).prefetch_related('tags') diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index f641298a..8f48929b 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -3,6 +3,7 @@ __package__ = 'archivebox.core' from django import forms from ..util import URL_REGEX +from .utils_taggit import edit_string_for_tags, parse_tags CHOICES = ( ('0', 'depth = 0 (archive just these URLs)'), @@ -12,3 +13,44 @@ CHOICES = ( class AddLinkForm(forms.Form): url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True) depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0') + + +class TagWidgetMixin: + def format_value(self, value): + if value is not None and not isinstance(value, str): + value = edit_string_for_tags(value) + return super().format_value(value) + +class TagWidget(TagWidgetMixin, forms.TextInput): + pass + +class TagField(forms.CharField): + widget = TagWidget + + def clean(self, value): + value = super().clean(value) + try: + return parse_tags(value) + except ValueError: + raise forms.ValidationError( + "Please provide a comma-separated list of tags." + ) + + def has_changed(self, initial_value, data_value): + # Always return False if the field is disabled since self.bound_data + # always uses the initial value in this case. + if self.disabled: + return False + + try: + data_value = self.clean(data_value) + except forms.ValidationError: + pass + + if initial_value is None: + initial_value = [] + + initial_value = [tag.name for tag in initial_value] + initial_value.sort() + + return initial_value != data_value diff --git a/archivebox/core/migrations/0006_auto_20200915_2006.py b/archivebox/core/migrations/0006_auto_20200915_2006.py deleted file mode 100644 index efb3d1d4..00000000 --- a/archivebox/core/migrations/0006_auto_20200915_2006.py +++ /dev/null @@ -1,90 +0,0 @@ -# Generated by Django 3.0.8 on 2020-09-15 20:06 - -from django.db import migrations, models -from django.contrib.contenttypes.models import ContentType -from django.utils.text import slugify -import django.db.models.deletion -import taggit.managers - -def forwards_func(apps, schema_editor): - SnapshotModel = apps.get_model("core", "Snapshot") - TaggedItemModel = apps.get_model("core", "TaggedItem") - TagModel = apps.get_model("taggit", "Tag") - contents = ContentType.objects.all() - try: - ct = ContentType.objects.filter(app_label="core", model="snapshot") - except model.DoesNotExist: # Be explicit about exceptions - ct = None - - db_alias = schema_editor.connection.alias - snapshots = SnapshotModel.objects.all() - for snapshot in snapshots: - tags = snapshot.tags - tag_set = ( - set(tag.strip() for tag in (snapshot.tags_old or '').split(',')) - ) - tag_set.discard("") - - for tag in tag_set: - new_tag, created = TagModel.objects.get_or_create(name=tag, slug=slugify(tag)) - TaggedItemModel.objects.get_or_create( - content_type_id=ct[0].id, - object_id=snapshot.id, - tag=new_tag - ) - - -def reverse_func(apps, schema_editor): - SnapshotModel = apps.get_model("core", "Snapshot") - TaggedItemModel = apps.get_model("core", "TaggedItem") - TagModel = apps.get_model("taggit", "Tag") - ct = ContentType.objects.get(app_label="core", model="snapshot") - - db_alias = schema_editor.connection.alias - snapshots = SnapshotModel.objects.all() - for snapshot in snapshots: - tags = TaggedItemModel.objects.filter( - object_id=snapshot.id, - ) - snapshot.tags_old = ",".join([tag.tag.name for tag in tags]) - snapshot.save() - - -class Migration(migrations.Migration): - - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('taggit', '0003_taggeditem_add_unique_index'), - ('core', '0005_auto_20200728_0326'), - ] - - operations = [ - migrations.RenameField( - model_name='snapshot', - old_name='tags', - new_name='tags_old', - ), - migrations.CreateModel( - name='TaggedItem', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('object_id', models.UUIDField(db_index=True, verbose_name='object ID')), - ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='core_taggeditem_tagged_items', to='contenttypes.ContentType', verbose_name='content type')), - ('tag', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='core_taggeditem_items', to='taggit.Tag')), - ], - options={ - 'verbose_name': 'Tag', - 'verbose_name_plural': 'Tags', - }, - ), - migrations.AddField( - model_name='snapshot', - name='tags', - field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='core.TaggedItem', to='taggit.Tag', verbose_name='Tags'), - ), - migrations.RunPython(forwards_func, reverse_func), - migrations.RemoveField( - model_name='snapshot', - name='tags_old', - ), - ] diff --git a/archivebox/core/migrations/0006_auto_20201012_1520.py b/archivebox/core/migrations/0006_auto_20201012_1520.py new file mode 100644 index 00000000..694c9908 --- /dev/null +++ b/archivebox/core/migrations/0006_auto_20201012_1520.py @@ -0,0 +1,70 @@ +# Generated by Django 3.0.8 on 2020-10-12 15:20 + +from django.db import migrations, models +from django.utils.text import slugify + +def forwards_func(apps, schema_editor): + SnapshotModel = apps.get_model("core", "Snapshot") + TagModel = apps.get_model("core", "Tag") + + db_alias = schema_editor.connection.alias + snapshots = SnapshotModel.objects.all() + for snapshot in snapshots: + tags = snapshot.tags + tag_set = ( + set(tag.strip() for tag in (snapshot.tags_old or '').split(',')) + ) + tag_set.discard("") + + for tag in tag_set: + to_add, _ = TagModel.objects.get_or_create(name=tag, slug=slugify(tag)) + snapshot.tags.add(to_add) + + +def reverse_func(apps, schema_editor): + SnapshotModel = apps.get_model("core", "Snapshot") + TagModel = apps.get_model("core", "Tag") + + db_alias = schema_editor.connection.alias + snapshots = SnapshotModel.objects.all() + for snapshot in snapshots: + tags = snapshot.tags.values_list("name", flat=True) + snapshot.tags_old = ",".join([tag for tag in tags]) + snapshot.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0005_auto_20200728_0326'), + ] + + operations = [ + migrations.RenameField( + model_name='snapshot', + old_name='tags', + new_name='tags_old', + ), + migrations.CreateModel( + name='Tag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True, verbose_name='name')), + ('slug', models.SlugField(max_length=100, unique=True, verbose_name='slug')), + ], + options={ + 'verbose_name': 'Tag', + 'verbose_name_plural': 'Tags', + }, + ), + migrations.AddField( + model_name='snapshot', + name='tags', + field=models.ManyToManyField(to='core.Tag'), + ), + migrations.RunPython(forwards_func, reverse_func), + migrations.RemoveField( + model_name='snapshot', + name='tags_old', + ), + ] diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 8ba0bb70..7d0c799f 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -2,22 +2,55 @@ __package__ = 'archivebox.core' import uuid -from django.db import models +from django.db import models, transaction from django.utils.functional import cached_property - -from taggit.managers import TaggableManager -from taggit.models import GenericUUIDTaggedItemBase, TaggedItemBase +from django.utils.text import slugify from ..util import parse_date from ..index.schema import Link +class Tag(models.Model): + """ + Based on django-taggit model + """ + name = models.CharField(verbose_name="name", unique=True, blank=False, max_length=100) + slug = models.SlugField(verbose_name="slug", unique=True, max_length=100) -class TaggedItem(GenericUUIDTaggedItemBase, TaggedItemBase): class Meta: verbose_name = "Tag" verbose_name_plural = "Tags" + def __str__(self): + return self.name + + def slugify(self, tag, i=None): + slug = slugify(tag) + if i is not None: + slug += "_%d" % i + return slug + + def save(self, *args, **kwargs): + if self._state.adding and not self.slug: + self.slug = self.slugify(self.name) + + with transaction.atomic(): + slugs = set( + type(self) + ._default_manager.filter(slug__startswith=self.slug) + .values_list("slug", flat=True) + ) + + i = None + while True: + slug = self.slugify(self.name, i) + if slug not in slugs: + self.slug = slug + return super().save(*args, **kwargs) + i = 1 if i is None else i+1 + else: + return super().save(*args, **kwargs) + class Snapshot(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) @@ -25,11 +58,10 @@ class Snapshot(models.Model): timestamp = models.CharField(max_length=32, unique=True, db_index=True) title = models.CharField(max_length=128, null=True, blank=True, db_index=True) - tags = TaggableManager(through=TaggedItem) added = models.DateTimeField(auto_now_add=True, db_index=True) updated = models.DateTimeField(null=True, blank=True, db_index=True) - # bookmarked = models.DateTimeField() + tags = models.ManyToManyField(Tag) keys = ('url', 'timestamp', 'title', 'tags', 'updated') @@ -113,3 +145,10 @@ class Snapshot(models.Model): and self.history['title'][-1].output.strip()): return self.history['title'][-1].output.strip() return None + + def save_tags(self, tags=[]): + tags_id = [] + for tag in tags: + tags_id.append(Tag.objects.get_or_create(name=tag)[0].id) + self.tags.clear() + self.tags.add(*tags_id) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 6ae2b6af..14b3b369 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -31,7 +31,6 @@ INSTALLED_APPS = [ 'core', 'django_extensions', - 'taggit', ] diff --git a/archivebox/core/utils_taggit.py b/archivebox/core/utils_taggit.py new file mode 100644 index 00000000..5a2d511d --- /dev/null +++ b/archivebox/core/utils_taggit.py @@ -0,0 +1,113 @@ +# Taken from https://github.com/jazzband/django-taggit/blob/3b56adb637ab95aca5036c37a358402c825a367c/taggit/utils.py + +def parse_tags(tagstring): + """ + Parses tag input, with multiple word input being activated and + delineated by commas and double quotes. Quotes take precedence, so + they may contain commas. + + Returns a sorted list of unique tag names. + + Ported from Jonathan Buchanan's `django-tagging + `_ + """ + if not tagstring: + return [] + + # Special case - if there are no commas or double quotes in the + # input, we don't *do* a recall... I mean, we know we only need to + # split on spaces. + if "," not in tagstring and '"' not in tagstring: + words = list(set(split_strip(tagstring, " "))) + words.sort() + return words + + words = [] + buffer = [] + # Defer splitting of non-quoted sections until we know if there are + # any unquoted commas. + to_be_split = [] + saw_loose_comma = False + open_quote = False + i = iter(tagstring) + try: + while True: + c = next(i) + if c == '"': + if buffer: + to_be_split.append("".join(buffer)) + buffer = [] + # Find the matching quote + open_quote = True + c = next(i) + while c != '"': + buffer.append(c) + c = next(i) + if buffer: + word = "".join(buffer).strip() + if word: + words.append(word) + buffer = [] + open_quote = False + else: + if not saw_loose_comma and c == ",": + saw_loose_comma = True + buffer.append(c) + except StopIteration: + # If we were parsing an open quote which was never closed treat + # the buffer as unquoted. + if buffer: + if open_quote and "," in buffer: + saw_loose_comma = True + to_be_split.append("".join(buffer)) + if to_be_split: + if saw_loose_comma: + delimiter = "," + else: + delimiter = " " + for chunk in to_be_split: + words.extend(split_strip(chunk, delimiter)) + words = list(set(words)) + words.sort() + return words + + +def split_strip(string, delimiter=","): + """ + Splits ``string`` on ``delimiter``, stripping each resulting string + and returning a list of non-empty strings. + + Ported from Jonathan Buchanan's `django-tagging + `_ + """ + if not string: + return [] + + words = [w.strip() for w in string.split(delimiter)] + return [w for w in words if w] + + +def edit_string_for_tags(tags): + """ + Given list of ``Tag`` instances, creates a string representation of + the list suitable for editing by the user, such that submitting the + given string representation back without changing it will give the + same list of tags. + + Tag names which contain commas will be double quoted. + + If any tag name which isn't being quoted contains whitespace, the + resulting string of tag names will be comma-delimited, otherwise + it will be space-delimited. + + Ported from Jonathan Buchanan's `django-tagging + `_ + """ + names = [] + for tag in tags: + name = tag.name + if "," in name or " " in name: + names.append('"%s"' % name) + else: + names.append(name) + return ", ".join(sorted(names)) diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 844ebbf4..aa7c8817 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -34,14 +34,19 @@ def remove_from_sql_main_index(snapshots: QuerySet, out_dir: Path=OUTPUT_DIR) -> def write_link_to_sql_index(link: Link): from core.models import Snapshot info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys} + tags = info.pop("tags") + if tags is None: + tags = [] + try: info["timestamp"] = Snapshot.objects.get(url=link.url).timestamp except Snapshot.DoesNotExist: while Snapshot.objects.filter(timestamp=info["timestamp"]).exists(): info["timestamp"] = str(float(info["timestamp"]) + 1.0) - Snapshot.objects.update_or_create(url=link.url, defaults=info) - return Snapshot.objects.get(url=link.url) + snapshot, _ = Snapshot.objects.update_or_create(url=link.url, defaults=info) + snapshot.save_tags(tags) + return snapshot @enforce_types @@ -72,9 +77,8 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None: ) tag_list = list(tag_set) or [] - for tag in tag_list: - snap.tags.add(tag) snap.save() + snap.save_tags(tag_list) diff --git a/setup.py b/setup.py index 0272f565..db83e9bf 100755 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ setuptools.setup( "base32-crockford==0.3.0", "django==3.0.8", "django-extensions==3.0.3", - "django-taggit==1.3.0", "dateparser", "ipython", diff --git a/tests/test_init.py b/tests/test_init.py index 72caa6d0..ae07e5da 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -157,18 +157,16 @@ def test_tags_migration(tmp_path, disable_extractors_dict): conn.row_factory = sqlite3.Row c = conn.cursor() c.execute(""" - SELECT snapshot.id snapshot, tags.name tag - FROM core_snapshot snapshot, core_taggeditem snapshot_tagged, taggit_tag tags - WHERE - snapshot.id = snapshot_tagged.object_id - AND tags.id = snapshot_tagged.tag_id + SELECT core_snapshot.id, core_tag.name from core_snapshot + JOIN core_snapshot_tags on core_snapshot_tags.snapshot_id=core_snapshot.id + JOIN core_tag on core_tag.id=core_snapshot_tags.tag_id """) tags = c.fetchall() conn.commit() conn.close() for tag in tags: - snapshot_id = tag['snapshot'] - tag_name = tag['tag'] + snapshot_id = tag["id"] + tag_name = tag["name"] # Check each tag migrated is in the previous field assert tag_name in snapshots_dict[snapshot_id] \ No newline at end of file