From 62c9028212f8c96d5faa99ea0061a011a4237249 Mon Sep 17 00:00:00 2001 From: Angel Rey Date: Mon, 21 Sep 2020 11:50:26 -0500 Subject: [PATCH] Improved tags --- archivebox.egg-info/requires.txt | 1 + archivebox/core/admin.py | 12 ++- .../migrations/0006_auto_20200915_2006.py | 89 ++++++++++++++++++ archivebox/core/models.py | 11 ++- archivebox/core/settings.py | 1 + archivebox/index/__init__.py | 11 ++- archivebox/index/schema.py | 3 +- archivebox/index/sql.py | 9 +- setup.py | 1 + tests/tags_migration/index.sqlite3 | Bin 0 -> 167936 bytes tests/test_init.py | 44 ++++++++- 11 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 archivebox/core/migrations/0006_auto_20200915_2006.py create mode 100755 tests/tags_migration/index.sqlite3 diff --git a/archivebox.egg-info/requires.txt b/archivebox.egg-info/requires.txt index 71dc253d..ca279875 100644 --- a/archivebox.egg-info/requires.txt +++ b/archivebox.egg-info/requires.txt @@ -4,6 +4,7 @@ mypy-extensions==0.4.3 base32-crockford==0.3.0 django==3.0.8 django-extensions==3.0.3 +django-taggit==1.3.0 dateparser ipython youtube-dl diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py index 4337e4a3..a35d589b 100644 --- a/archivebox/core/admin.py +++ b/archivebox/core/admin.py @@ -66,6 +66,12 @@ class SnapshotAdmin(admin.ModelAdmin): actions = [delete_snapshots, overwrite_snapshots, update_snapshots, update_titles, verify_snapshots] actions_template = 'admin/actions_as_select.html' + def get_queryset(self, request): + return super().get_queryset(request).prefetch_related('tags') + + def tag_list(self, obj): + return u", ".join(o.name for o in obj.tags.all()) + def id_str(self, obj): return format_html( '{}', @@ -75,9 +81,9 @@ class SnapshotAdmin(admin.ModelAdmin): def title_str(self, obj): canon = obj.as_link().canonical_outputs() tags = ''.join( - format_html('{}', tag.strip()) - for tag in obj.tags.split(',') - ) if obj.tags else '' + format_html(' {} ', tag) + for tag in obj.tags.all() + ) if obj.tags.all() else '' return format_html( '' '' diff --git a/archivebox/core/migrations/0006_auto_20200915_2006.py b/archivebox/core/migrations/0006_auto_20200915_2006.py new file mode 100644 index 00000000..59bb111e --- /dev/null +++ b/archivebox/core/migrations/0006_auto_20200915_2006.py @@ -0,0 +1,89 @@ +# Generated by Django 3.0.8 on 2020-09-15 20:06 + +from django.db import migrations, models +from django.contrib.contenttypes.models import ContentType +from django.utils.text import slugify +import django.db.models.deletion +import taggit.managers + +def forwards_func(apps, schema_editor): + SnapshotModel = apps.get_model("core", "Snapshot") + TaggedItemModel = apps.get_model("core", "TaggedItem") + TagModel = apps.get_model("taggit", "Tag") + contents = ContentType.objects.all() + try: + ct = ContentType.objects.filter(app_label="core", model="snapshot") + except model.DoesNotExist: # Be explicit about exceptions + ct = None + + db_alias = schema_editor.connection.alias + snapshots = SnapshotModel.objects.all() + for snapshot in snapshots: + tags = snapshot.tags + tag_set = ( + set(tag.strip() for tag in (snapshot.tags_old or '').split(',')) + ) + tag_list = list(tag_set) or [] + + for tag in tag_list: + new_tag, created = TagModel.objects.get_or_create(name=tag, slug=slugify(tag)) + TaggedItemModel.objects.get_or_create( + content_type_id=ct[0].id, + object_id=snapshot.id, + tag=new_tag + ) + + +def reverse_func(apps, schema_editor): + SnapshotModel = apps.get_model("core", "Snapshot") + TaggedItemModel = apps.get_model("core", "TaggedItem") + TagModel = apps.get_model("taggit", "Tag") + ct = ContentType.objects.get(app_label="core", model="snapshot") + + db_alias = schema_editor.connection.alias + snapshots = SnapshotModel.objects.all() + for snapshot in snapshots: + for tag in tags: + tagged_items = TaggedItemModel.objects.filter( + object_id=snapshot.id, + ).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('taggit', '0003_taggeditem_add_unique_index'), + ('core', '0005_auto_20200728_0326'), + ] + + operations = [ + migrations.RenameField( + model_name='snapshot', + old_name='tags', + new_name='tags_old', + ), + migrations.CreateModel( + name='TaggedItem', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.UUIDField(db_index=True, verbose_name='object ID')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='core_taggeditem_tagged_items', to='contenttypes.ContentType', verbose_name='content type')), + ('tag', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='core_taggeditem_items', to='taggit.Tag')), + ], + options={ + 'verbose_name': 'Tag', + 'verbose_name_plural': 'Tags', + }, + ), + migrations.AddField( + model_name='snapshot', + name='tags', + field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='core.TaggedItem', to='taggit.Tag', verbose_name='Tags'), + ), + migrations.RunPython(forwards_func, reverse_func), + migrations.RemoveField( + model_name='snapshot', + name='tags_old', + ), + ] diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 313dd67d..b7719b2e 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -5,10 +5,19 @@ import uuid from django.db import models from django.utils.functional import cached_property +from taggit.managers import TaggableManager +from taggit.models import GenericUUIDTaggedItemBase, TaggedItemBase + from ..util import parse_date from ..index.schema import Link + +class TaggedItem(GenericUUIDTaggedItemBase, TaggedItemBase): + class Meta: + verbose_name = "Tag" + verbose_name_plural = "Tags" + class Snapshot(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) @@ -16,7 +25,7 @@ class Snapshot(models.Model): timestamp = models.CharField(max_length=32, unique=True, db_index=True) title = models.CharField(max_length=128, null=True, blank=True, db_index=True) - tags = models.CharField(max_length=256, null=True, blank=True, db_index=True) + tags = TaggableManager(through=TaggedItem) added = models.DateTimeField(auto_now_add=True, db_index=True) updated = models.DateTimeField(null=True, blank=True, db_index=True) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 14b3b369..6ae2b6af 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -31,6 +31,7 @@ INSTALLED_APPS = [ 'core', 'django_extensions', + 'taggit', ] diff --git a/archivebox/index/__init__.py b/archivebox/index/__init__.py index 06832dbc..f93a4ab8 100644 --- a/archivebox/index/__init__.py +++ b/archivebox/index/__init__.py @@ -86,9 +86,16 @@ def merge_links(a: Link, b: Link) -> Link: ) # all unique, truthy tags + tags_a = [] + if a.tags: + tags_a = a.tags.all() + tags_b = [] + if b.tags: + tags_b = b.tags.all() + tags_set = ( - set(tag.strip() for tag in (a.tags or '').split(',')) - | set(tag.strip() for tag in (b.tags or '').split(',')) + set(tag.name.strip() for tag in tags_a) + | set(tag.name.strip() for tag in tags_b) ) tags = ','.join(tags_set) or None diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py index 7508890d..7ed44e74 100644 --- a/archivebox/index/schema.py +++ b/archivebox/index/schema.py @@ -157,7 +157,8 @@ class Link: assert isinstance(self.url, str) and '://' in self.url assert self.updated is None or isinstance(self.updated, datetime) assert self.title is None or (isinstance(self.title, str) and self.title) - assert self.tags is None or isinstance(self.tags, str) + #for tag in self.tags.all(): + # assert tag is None or isinstance(tag, TaggedItem) assert isinstance(self.sources, list) assert all(isinstance(source, str) and source for source in self.sources) assert isinstance(self.history, dict) diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index b3ca7231..bd3664da 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -65,7 +65,14 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None: except Snapshot.DoesNotExist: snap = write_link_to_sql_index(link) snap.title = link.title - snap.tags = link.tags + + tag_set = ( + set(tag.strip() for tag in (link.tags or '').split(',')) + ) + tag_list = list(tag_set) or [] + + for tag in tag_list: + snap.tags.add(tag) snap.save() diff --git a/setup.py b/setup.py index db83e9bf..0272f565 100755 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ setuptools.setup( "base32-crockford==0.3.0", "django==3.0.8", "django-extensions==3.0.3", + "django-taggit==1.3.0", "dateparser", "ipython", diff --git a/tests/tags_migration/index.sqlite3 b/tests/tags_migration/index.sqlite3 new file mode 100755 index 0000000000000000000000000000000000000000..04d35a71e68e8460936ae8f525bcfc169e53e967 GIT binary patch literal 167936 zcmeI5du$xXeaClik31g9yU~N>*%l?OC{fgbqPUNH(s629p)JOiWb0u^F^ui+`bYx!k0J;XBZ!L@ND$OPizY>zf6}B)(FSPz zM^d0OvwOFXJ3gc+jxBx<;_7zxH^2GLXC8Mmvp0M3sV7QJOJ1$jD`r#Hg*}2O3J=S& zAPBG1zohdg(*Iwke_{I9d9kqVUat$MCjV-HT8ZSR%r;^Fw_P+`2LTWO0T2KI5C8!X z009sH0T2KI5V(5;#t+4)vx(sw0{J@m1Sygu@o&d}JN`)Q|6+d=yB150et-1Sqfd|S z9r?z{r$$yrjt~EE_%En2ULXJhAOHd&00JNY0w8d^1oVjk;o$UQ!)i22wQ56A6xAqI zOU;s5))Y;dQ}T0aR#tNhYHC5xoJyy%ik@E>9}vbLTrAY;7LAZL%vQ5zFmolVLT3y)hjjMa&W^~0+6+P8IuA0_T3;VorHE&!k zP1DuXzH!xjI+vT@OXE&=^1C%4HI>Tg5AUH79$Tbb)+tccXl)eDre$o@Yu9fW8&%)j+Bg^el$iMUF-ntm_$c-BT*pt=@pIKImnffF zy~Rh!Kl8&<=h z^|Nn%sq+D-~PmCh=v zc4lxun0}1bb{8X~NcmQ3SG#x(#+N(oTS947S98k)144poDppEW>OwatPg*kd8@!y& zLX&nm#%kGIGYadqQel(A)^quswiu?aC}qc$=DKoZ@ji%FpT25zm32uHbury_v1Kz( zX?ix3oexo_GzM?RTX&2pHKU}ptVq$Zj>3m->y(s~p3`IGAA~_t7s(IEXGxuWnS7Fb z{C2x?*nfBrM0$FJox00JK zA#@Dy=Pn1i%g&JS-j*CFY5pM)!opBVvfV;KT4CP9Ws?;6Zfl zpP~;rPI%BDPdUWA@biJOaCBDWUfj`pcp@C7clFz)>jq0a*zx8D)Aj#oGRi!56?Yn z7Fy+|S(J;GY=>F(H$G=B$V+s)L$fZMWw}8&Rjo-Ak{-QWSZ`f5oB|9?gxpCR8OKkl0#0zm)-KmY_l00ck)1V8`;KmY_l00e&F1m-0xI<-<-lj)Nx z%&C)dprbnvpE{SxYPE+nMN{UK{G6JV)!c%nEu^$lS~io*Dtre)et6ibn5FWmLaowct>#W; z^17m@+3){F^0q+#;ROO900JNY0w4eaAOHd&00JNY0w8d25g3vZbR)n@sfzpm@2!GG zZa@G8KmY_l00ck)1V8`;KmY_lpdSH@|NBuv7zls>2!H?xfB*=900@8p2!H?x+*<_L z{r{8XIf49)yiI;genh@c{*C-Id5e65{4IHde2KhH{+j$5`4jSK@`vR2$nTI>Nt@gx zEm9*TQlJs>0s#;J0T2KI5C8!X009sH0T2Lz-A-UY62-_QKOW%62l(-Rb{xEq9~1mI z!H?tYI55VK`}uJnKkjA6@E(37{21rQ7&}U%{5Zmo!~7U!$IuWzM)+}%9|yuxR1C9& z#10`zibk>izuPM*@(%(a00JNY0w4eaAOHd&00JPe8wjxVe?0$hH!3m00ck)1V8`;KmY_l00edm0gV55OGzW2 zAOHd&00JNY0w4eaAOHd&00O(2KrHgV0ujG1kYn+0Mc*3yYV6wR#|Fm+o(cc{$lK9h z8Cn{7pY*2q_0W0oHQ}4WLjh7F|26p0KoCxUDFj~r$e~DLZcco4zG<$Mt>W`$b**NY z#Y(Aalxu6j^Mj`^EG=GIk}oYj^2Cxn6}&tppO`8Yr{q$#X{}jx`P}(S^0~`TJRv`M zVfpOhg)8#oOIPH@%a_hCpQDb>E}gq{Ql2sk%~GvuG)on0N-mmBi@obcr3ST?=dD81 zV2+#C^(Ob|2G^~P`jmXttQXeJ`Uy=@lHRy9u)1a$6|2!O*QkHnfg_7LDb%RcYSU=m z*sypCUFl5rg{4QAE-alpy>zjgS8oI-yGi*2%XHPh4>Vmp(vKX*I06 z-+Kp)HwClRTsN5Mj{Q35R?Ftvl-#J8ritQIm)uMI1tAPtKHmrK3)M(K1X?Wh#tQhL*%1U}wD{LO&0jI7kpZmbl zhiEZ(a<@~@Q}X$9TSa4(Tzw30vi&RjB8eB~#C9W)nYDVYwb8IsrsDZ2%SAq=r^Z3e7}={%jcG#y1XR!9QAdx{^#w0GZT9xMzD^bBjrH^O3*Aia zz%jRV)Nj+1B}a}6Pu|UssrI$;p+q~nDBhZPYYd0N58f*8pbAt}HDzX2GP)LMzron| zZPSv2_T7%cRlxf^Z)%>3#-j&)1e&D3ZwM#aFU!ld!M4@*7-9PiVAH^XM_Uqejw|cXul@ z?RMQvvE1*#y&GYN4Td(BiQWd&&P_xT1^StLB~bXmI@GO)W(dp z!ycc(iV=+9t;ostp8b);d0KDPfOK9;?k8-T2S4p3cJ4qTE7cZCWW5buGQ3xIIWu)IE;oKHmd+Y~6;tMKYdD^*CTzvjb_B z?5U&4TOS&YBo-INR~~d*C@-*Jscl|UVtra_Ul&dt?`;=beVgZ=cGjAhsguSwA3QxA zdNuWYV7mNwmPYc)&|qSAR=mQ&HeM{3nwHV9UTj&_f)zNGdXMw8ugQmT5V8|hsb zbn>^XaN5L9ej8E3VSn)Zisv)1RjW*mowCCd#w!m*uN0$@Z}sy0AX$TUdAJw+-(-DQrzc zTTNv%R?1#{aqZ&%X2lEIy}GFv&uWEh%_>14S=zuZ9*-n$%!;>&yAb0>=}DkaD_Shq zhN7A)dNyskYcOuCK}U3-#TVCThpu;=6Iv!6ar8cc>FEet7MJw|1(MUqq#ansr z!i_g(9}NTkqVHpIN5${z+q6_{)%`Ei64lblKDQm^d1PvygZ23Qo zH;?aw-8ExbXZSMdYWvHKl9Y+nQ4=1uJK*1f?C=Mw++4Sie|s{A?~U z5MwGyzq022pU*oy%eMv-XJ4J$ma zW+ahJinnIGt=;*$>eM;s?3f>$|01j7oiDjBST|hBDbII|0K<+w6>Ik1o$CGi+Gh>+ z!CxOtoKJ36$Nb_9)qj4x9sadQVtGbvPxD6Ts{w{X&(QT^x~LYizN<(4&CYkNC#K!j zd^Wr;`qj|yJAhNi9N^kSyy^C54g!5?oE1$Ny@-n<@3oTlmD-zR-m z@r*(tmr1MnlrQ`^62Z`Vu1V8`;KmY_l00ck)1V8`;KmY{pJpyd}A0qS_06(P% zyg&d1KmY_l00ck)1V8`;KmY_l00izG0t1pLMnYjpj7IG7|Jwrj8F~BO$slq90w4ea zAOHd&00JNY0w4eaAOHd&KnaAUs2Gx@Xf#GoLh;`eNQxft0s#;J0T2KI5C8!X009sH z0T2LzUl4(|8X6WBFSo}picwKhG-Xc7Ki%`Is9LMnSoQ}TK`omNtosb*H^%++je zE|pVqb2)9rnp-WHX*H$k+0<(KSv8|5dM1;~XHO|JP1HkZ$8xw%!7J|`)i z$rV>krK35M&(T;)E|an~Q^_T&qFORrMa!gmdZ4lI<|7{>JCb z1$n93tebV&EX$2jvsBwzUfB45a^$N5`6u${L{~iBk{EP9|;v4Y`@tN39 zVqc5>LF{VmTWcA;|q32q4;MpO^ z_t|zGcCK9qogE_YrFKVW2RJ#`h=a3FXZKKxxonuLavo>dys=?*l{sND3!Ep^SbZ?; z$P1V|5`7Wkb_7RfAcCXS8zC8^5oXh2COBZtM0-sqN2%%I8Hp=r&kF5IW+(Cpy?S(3 za-?>yI~uvgux;UJ>sUCV*y#gNYH@H{Vq!X%nF{v)&=9?LBpKqV^QVG$ljmns5qf=k zF65-&wQvO3HiNc}qo8Z!NZ{v_1Jvg5Oo(ad8ZZ(3WF|ZyL}zs_!hLLTSBV=mD^bJw ztfR&aXQqxMH*hw@jU8ETI5T#%xq;InHBPEbpBu<5nMgN;-TyD_c?Y{IBn1K>00JNY z0w4eaAOHd&00JNY0wC~?6R>~(|As)mN8TWR{EjD&1VI1|bm-xH{6UeTn3(qh#A!wNOEkfuM8$fR?*j6MFh_x~e5 z?(vNP5C8!X009sH0T2KI5C8!X009sHf%htbVY>gHAWD)XL}TP7Ary}b@p$}8@oVqZ z3_&*tfB*=900@8p2!H?xfB*=9!0i!epA?7AF3#)LqR`oKk8QZe_S<9I?X7xt+uM9+ zy`F9O`fRi}wNhG>i)PcDIw=>*wN~+@OgHAM*66MbbT7M;aN!f?&G&A1k3DCPzLWBlHCMLGdX)y) z`JwqLT1~TDwu1USYxA^lUcS8oy6e`%