exclude links that are in blacklist

archivebox/links.py

@@ -28,13 +28,19 @@ from util import (
     check_links_structure,
 )
 
+from config import (
+    URL_BLACKLIST,
+)
 
 def validate_links(links):
     check_links_structure(links)
     links = archivable_links(links)  # remove chrome://, about:, mailto: etc.
     links = uniquefied_links(links)  # merge/dedupe duplicate timestamps & urls
     links = sorted_links(links)      # deterministically sort the links based on timstamp, url
-
+    links = exclude_links(links)     # exclude links that are in blacklist
+    
+    print(links)
+     
     if not links:
         print('[X] No links found :(')
         raise SystemExit(1)
@@ -42,7 +48,8 @@ def validate_links(links):
     for link in links:
         link['title'] = unescape(link['title'].strip()) if link['title'] else None
         check_link_structure(link)
-
+    
+    print("FINAL LIST", list(links))
     return list(links)
 
 
@@ -115,3 +122,10 @@ def lowest_uniq_timestamp(used_timestamps, timestamp):
         new_timestamp = '{}.{}'.format(timestamp, nonce)
 
     return new_timestamp
+
+def exclude_links(links):
+    """ exclude links that are in blacklist"""
+
+    links = [link for link in links if not URL_BLACKLIST.match(link['url'])]
+
+    return links