mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-04-06 07:47:53 +10:00
continue renaming extractor to plugin, add plan for hook concurrency, add chrome kill helper script
This commit is contained in:
@@ -25,7 +25,7 @@ from urllib.request import urlopen
|
||||
|
||||
import rich_click as click
|
||||
|
||||
EXTRACTOR_NAME = 'parse_txt_urls'
|
||||
PLUGIN_NAME = 'parse_txt_urls'
|
||||
|
||||
# URL regex from archivebox/misc/util.py
|
||||
# https://mathiasbynens.be/demo/url-regex
|
||||
@@ -127,7 +127,7 @@ def main(url: str, snapshot_id: str = None):
|
||||
f.write(json.dumps({
|
||||
'type': 'Snapshot',
|
||||
'url': found_url,
|
||||
'via_extractor': EXTRACTOR_NAME,
|
||||
'plugin': PLUGIN_NAME,
|
||||
}) + '\n')
|
||||
|
||||
click.echo(f'Found {len(urls_found)} URLs')
|
||||
|
||||
@@ -186,7 +186,7 @@ https://other.com
|
||||
entry = json.loads(output_file.read_text().strip())
|
||||
assert entry['url'] == 'https://example.com'
|
||||
assert 'type' in entry
|
||||
assert 'via_extractor' in entry
|
||||
assert 'plugin' in entry
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user