forks/archivebox
1
0
Fork 0
Code Releases Activity

cleanup update flag handling and show better logging to clarify when its working

Browse source
This commit is contained in:
Nick Sweeting 2022-05-09 20:15:55 -07:00
parent 38e54b93fe
commit 8cfe6f4afb
2 changed files with 23 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
archivebox/cli/archivebox_add.py
View file

@ -30,11 +30,17 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3", help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
) )
parser.add_argument( parser.add_argument(
'--update-all', #'-n', '--update', #'-u',
action='store_true', action='store_true',
default=not ONLY_NEW, # when ONLY_NEW=True we skip updating old links default=not ONLY_NEW, # when ONLY_NEW=True we skip updating old links
help="Also retry previously skipped/failed links when adding new links", help="Also retry previously skipped/failed links when adding new links",
) )
parser.add_argument(
'--update-all', #'-n',
action='store_true',
default=False,
help="Also update ALL links in index when finished adding new links",
)
parser.add_argument( parser.add_argument(
'--index-only', #'-o', '--index-only', #'-o',
action='store_true', action='store_true',
@ -104,6 +110,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
urls=stdin_urls or urls, urls=stdin_urls or urls,
depth=command.depth, depth=command.depth,
tag=command.tag, tag=command.tag,
update=command.update,
update_all=command.update_all, update_all=command.update_all,
index_only=command.index_only, index_only=command.index_only,
overwrite=command.overwrite, overwrite=command.overwrite,

18
archivebox/main.py
View file

@ -4,8 +4,9 @@ import os
import sys import sys
import shutil import shutil
import platform import platform
from django.utils import timezone
from pathlib import Path from pathlib import Path
from datetime import date from datetime import date, datetime
from typing import Dict, List, Optional, Iterable, IO, Union from typing import Dict, List, Optional, Iterable, IO, Union
from crontab import CronTab, CronSlices from crontab import CronTab, CronSlices
@ -554,7 +555,8 @@ def oneshot(url: str, extractors: str="", out_dir: Path=OUTPUT_DIR):
def add(urls: Union[str, List[str]], def add(urls: Union[str, List[str]],
tag: str='', tag: str='',
depth: int=0, depth: int=0,
update_all: bool=not ONLY_NEW, update: bool=not ONLY_NEW,
update_all: bool=False,
index_only: bool=False, index_only: bool=False,
overwrite: bool=False, overwrite: bool=False,
# duplicate: bool=False, # TODO: reuse the logic from admin.py resnapshot to allow adding multiple snapshots by appending timestamp automatically # duplicate: bool=False, # TODO: reuse the logic from admin.py resnapshot to allow adding multiple snapshots by appending timestamp automatically
@ -621,11 +623,21 @@ def add(urls: Union[str, List[str]],
if extractors: if extractors:
archive_kwargs["methods"] = extractors archive_kwargs["methods"] = extractors
if update_all: stderr()
ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
if update:
stderr(f'[*] [{ts}] Archiving + updating', len(imported_links), 'URLs from added set...', color='green')
archive_links(imported_links, overwrite=overwrite, **archive_kwargs)
elif update_all:
stderr(f'[*] [{ts}] Archiving + updating ', len(all_links), 'URLs from entire library...', color='green')
archive_links(all_links, overwrite=overwrite, **archive_kwargs) archive_links(all_links, overwrite=overwrite, **archive_kwargs)
elif overwrite: elif overwrite:
stderr(f'[*] [{ts}] Archiving + overwriting', len(imported_links), 'URLs from added set...', color='green')
archive_links(imported_links, overwrite=True, **archive_kwargs) archive_links(imported_links, overwrite=True, **archive_kwargs)
elif new_links: elif new_links:
stderr(f'[*] [{ts}] Archiving', len(imported_links), 'URLs from added set...', color='green')
archive_links(new_links, overwrite=False, **archive_kwargs) archive_links(new_links, overwrite=False, **archive_kwargs)