forks/archivebox
1
0
Fork 0
Code Releases Activity

fix lint errors

Browse source
This commit is contained in:
Nick Sweeting 2021-04-06 02:50:51 -04:00
parent e92db03f5f
commit 21bb84f627
2 changed files with 6 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
archivebox/search/backends/sonic.py
View file

@ -13,10 +13,13 @@ MAX_SONIC_TEXT_CHUNK_LENGTH = 2000 # dont index more than 2000 characte
def index(snapshot_id: str, texts: List[str]): def index(snapshot_id: str, texts: List[str]):
with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl: with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl:
for text in texts: for text in texts:
max_length = 1000000
chunks = ( chunks = (
text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH] text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH]
for i in range(0, min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH), MAX_SONIC_TEXT_CHUNK_LENGTH) for i in range(
0,
min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH),
MAX_SONIC_TEXT_CHUNK_LENGTH,
)
) )
for chunk in chunks: for chunk in chunks:
ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk)) ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))

2
archivebox/system.py
View file

@ -68,7 +68,7 @@ def run(*args, input=None, capture_output=True, timeout=None, check=False, text=
# force kill any straggler subprocesses that were forked from the main proc # force kill any straggler subprocesses that were forked from the main proc
try: try:
os.killpg(pgid, signal.SIGINT) os.killpg(pgid, signal.SIGINT)
except Exception as e: except Exception:
pass pass
return CompletedProcess(process.args, retcode, stdout, stderr) return CompletedProcess(process.args, retcode, stdout, stderr)