More OG tweaks

This commit is contained in:
Thomas Sileo 2019-08-06 21:47:02 +02:00
parent b0cb248a23
commit fa3d2ca288

View file

@ -59,6 +59,9 @@ def fetch_og_metadata(user_agent, links):
except requests.HTTPError as http_err:
logger.debug(f"failed to HEAD {l}, got a {http_err.response.status_code}")
continue
except requests.Timeout:
logger.debug(f"HEAD {l} timed out")
continue
if not h.headers.get("content-type").startswith("text/html"):
logger.debug(f"skipping {l} for bad content type")
@ -70,6 +73,9 @@ def fetch_og_metadata(user_agent, links):
except requests.HTTPError as http_err:
logger.debug(f"failed to GET {l}, got a {http_err.response.status_code}")
continue
except requests.Timeout:
logger.debug(f"GET {l} timed out")
continue
r.encoding = "UTF-8"
html = r.text