formatting + waybackpy
This commit is contained in:
parent
9552369760
commit
d33ced1214
6 changed files with 14 additions and 16 deletions
17
archive.py
17
archive.py
|
|
@ -1,16 +1,15 @@
|
|||
import requests, time, random
|
||||
from waybackpy import WaybackMachineSaveAPI
|
||||
import time
|
||||
import random
|
||||
|
||||
from config import ARCHIVE_URLS, USER_AGENT
|
||||
|
||||
def archive_url(url):
|
||||
print(f"🌐 Archiving {url} ...")
|
||||
headers = {"User-Agent": USER_AGENT}
|
||||
try:
|
||||
resp = requests.get(f"https://web.archive.org/save/{url}", headers=headers, timeout=30)
|
||||
if resp.status_code == 200:
|
||||
print(f"✅ Archived {url}")
|
||||
else:
|
||||
print(f"⚠️ Failed to archive {url}, status code {resp.status_code}")
|
||||
save_api = WaybackMachineSaveAPI(url, user_agent=USER_AGENT)
|
||||
save_api.save()
|
||||
print(f"✅ Archived {url}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Exception archiving {url}: {e}")
|
||||
|
||||
|
|
@ -19,3 +18,7 @@ def archive_all_urls():
|
|||
delay = 10 + random.uniform(-3, 3)
|
||||
time.sleep(delay)
|
||||
archive_url(url)
|
||||
|
||||
def test_archive():
|
||||
test_url = "https://httpbin.org/anything/foo/bar"
|
||||
archive_url(test_url)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue