forks/archivebox
1
0
Fork 0
Code Releases Activity

link docstring

Browse source
This commit is contained in:
Nick Sweeting 2017-10-18 19:08:33 -05:00
parent d74394fa45
commit 7e4ca8a6ac
1 changed files with 37 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
links.py
View file

@ -5,6 +5,39 @@ from util import (
get_link_type, get_link_type,
) )
"""
In Bookmark Archiver, a Link represents a single entry that we track in the
json index. All links pass through all archiver functions and the latest,
most up-to-date canonical output for each is stored in "latest_archives".
.
Link {
timestamp: float, (how we uniquely id links) _ _ _ _ ___
url: str, | \ / \ |\| ' |
base_url: str, |_/ \_/ | | |
domain: str, _ _ _ _ _ _
tags: str, |_) /| |\| | / `
type: str, | /"| | | | \_,
title: str, ,-'"`-.
sources: [str], /// / @ @ \ \\\\
latest_archives: { :=| ,._,. |=: /
..., || ,\ \_../ /. ||
pdf: 'output.pdf', ||','`-._))'`.`||
wget: 'example.com/1234/index.html' `-' (/ `-'
},
history: {
...
pdf: [
{timestamp: 15444234325, status: 'skipped', result='output.pdf'},
...
],
wget: [
{timestamp: 11534435345, status: 'succeded', result='donuts.com/eat/them.html'}
]
},
}
"""
def validate_links(links): def validate_links(links):
links = valid_links(links) # remove chrome://, about:, mailto: etc. links = valid_links(links) # remove chrome://, about:, mailto: etc.
@ -25,18 +58,20 @@ def sorted_links(links):
) )
def merge_links(link1, link2): def merge_links(link1, link2):
"""deterministially merge two links, favoring longer field values over shorter,
and "cleaner" values over worse ones.
"""
longer = lambda a, b, key: a[key] if len(a[key]) > len(b[key]) else b[key] longer = lambda a, b, key: a[key] if len(a[key]) > len(b[key]) else b[key]
earlier = lambda a, b, key: a[key] if a[key] < b[key] else b[key] earlier = lambda a, b, key: a[key] if a[key] < b[key] else b[key]
url = longer(link1, link2, 'url') url = longer(link1, link2, 'url')
earliest_ts = earlier(link1, link2, 'timestamp')
longest_title = longer(link1, link2, 'title') longest_title = longer(link1, link2, 'title')
cleanest_title = link1['title'] if '://' not in link1['title'] else link2['title'] cleanest_title = link1['title'] if '://' not in link1['title'] else link2['title']
link = { link = {
'url': url, 'url': url,
'domain': domain(url), 'domain': domain(url),
'base_url': base_url(url), 'base_url': base_url(url),
'timestamp': earliest_ts, 'timestamp': earlier(link1, link2, 'timestamp'),
'tags': longer(link1, link2, 'tags'), 'tags': longer(link1, link2, 'tags'),
'title': longest_title if '://' not in longest_title else cleanest_title, 'title': longest_title if '://' not in longest_title else cleanest_title,
'sources': list(set(link1['sources'] + link2['sources'])), 'sources': list(set(link1['sources'] + link2['sources'])),