alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-03-30 03:42:27 +10:00
Code Issues Packages Projects Releases Wiki Activity

fix: Bugs related to add() refactor

Browse Source
This commit is contained in:
Cristian
2020-07-13 14:48:25 -05:00
parent a79dd4685a
commit 5e2bf73f04
2 changed files with 9 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
archivebox/index/__init__.py
View File

@@ -292,7 +292,6 @@ def dedupe_links(existing_links: List[Link],
new_links: List[Link]) -> Tuple[List[Link], List[Link]]:
from ..parsers import parse_links
# merge existing links in out_dir and new links
all_links = validate_links(existing_links + new_links)
all_link_urls = {link.url for link in existing_links}
@@ -301,6 +300,11 @@ def dedupe_links(existing_links: List[Link],
link for link in new_links
if link.url not in all_link_urls
]
all_links_deduped = {link.url: link for link in all_links}
for i in range(len(new_links)):
if new_links[i].url in all_links_deduped.keys():
new_links[i] = all_links_deduped[new_links[i].url]
log_deduping_finished(len(new_links))
return all_links, new_links