Commit 01ff7ab4 authored by Romain Courteaud's avatar Romain Courteaud

Add SNI support

parent 822ac1a4
# URL checker contribution
## Install development environnment
virtualenv venv
. venv/bin/activate
pip install black pyflakes
pip install --upgrade --editable .
## Check the code
pyflakes *py
black -t py37 -l 79 *py
## SNI Support
https://stackoverflow.com/questions/18578439/using-requests-with-tls-doesnt-give-sni-support/18579484
\ No newline at end of file
......@@ -8,11 +8,11 @@ setup(
include_package_data=False,
zip_safe=True,
install_requires=[
"setuptools==40.5.0",
"requests==2.20.0",
"requests_toolbelt==0.8.0",
"peewee==2.10.1",
"click==7.0",
"setuptools>40.5.0",
"requests>2.20.0",
"forcediphttpsadapter",
"peewee>2.10.1",
"click>=7.0",
],
entry_points={
"console_scripts": ["urlchecker=urlchecker_cli:runUrlChecker "]
......
from urllib.parse import urlparse
from urllib.parse import urlparse, urlunsplit
import requests
import socket
import sys
import traceback
import time
from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
__version__ = "0.0.1"
__version__ = "0.0.2"
PREFERRED_TYPE = "text/html"
TIMEOUT = 2
......@@ -31,6 +32,8 @@ class WebBot:
stream=False,
timeout=TIMEOUT,
allow_redirects=False,
verify=True,
session=requests,
**kwargs,
):
......@@ -45,11 +48,13 @@ class WebBot:
kwargs["stream"] = stream
kwargs["timeout"] = timeout
kwargs["allow_redirects"] = allow_redirects
args = [url]
kwargs["verify"] = verify
args = [method, url]
kwargs["headers"] = headers
try:
response = requests.get(*args, **kwargs)
response = session.request(*args, **kwargs)
except requests.exceptions.SSLError:
# XXX Enter into unknown host
response = requests.models.Response()
......@@ -72,8 +77,9 @@ class WebBot:
return response
def check(self, url):
print(url)
parsed_url = urlparse(url)
response = self.request("GET", url)
print(url, response.status_code)
# Get the list of available IPv4 frontend CDN
hostname = parsed_url.hostname
......@@ -81,12 +87,20 @@ class WebBot:
ip_list = [x[4][0] for x in dns_info_list]
for ip in ip_list:
parsed_url = parsed_url._replace(netloc=ip)
ip_url = parsed_url.geturl()
response = self.request("GET", ip_url, headers={"Host": hostname})
print(ip, response.status_code)
session = requests.Session()
# SNI Support
if parsed_url.scheme == "https":
# Provide SNI support
base_url = urlunsplit(
(parsed_url.scheme, parsed_url.netloc, "", "", "")
)
session.mount(base_url, ForcedIPHTTPSAdapter(dest_ip=ip))
response = self.request(
"GET", url, headers={"Host": hostname}, session=session
)
print(ip, hostname, response.status_code)
def run(self, url):
print(time.strftime("%Y-%m-%d %H:%M:%S"))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment