Add missing files.

Makefile

+.PHONY: all develop test lint clean doc format
+
+
+# The package name
+PKG=FIXME
+
+
+all: test lint
+
+#
+# Setup
+#
+develop: install-deps activate-pre-commit configure-git
+
+install-deps:
+	@echo "--> Installing dependencies"
+	pip install -U pip setuptools wheel
+	poetry install
+
+activate-pre-commit:
+	@echo "--> Activating pre-commit hook"
+	pre-commit install
+
+configure-git:
+	@echo "--> Configuring git"
+	git config branch.autosetuprebase always
+
+#
+# testing & checking
+#
+test-all: test
+
+test:
+	@echo "--> Running Python tests"
+	pytest --ff -x -p no:randomly
+	@echo ""
+
+test-randomly:
+	@echo "--> Running Python tests in random order"
+	pytest
+	@echo ""
+
+test-with-coverage:
+	@echo "--> Running Python tests"
+	py.test --cov $(PKG)
+	@echo ""
+
+test-with-typeguard:
+	@echo "--> Running Python tests with typeguard"
+	pytest --typeguard-packages=${PKG}
+	@echo ""
+
+#
+# Various Checkers
+#
+lint: lint-py lint-mypy lint-rst lint-doc
+
+lint-ci: lint
+
+lint-all: lint lint-bandit
+
+lint-py:
+	@echo "--> Linting Python files /w flake8"
+	flake8 src tests
+	@echo ""
+
+lint-mypy:
+	@echo "--> Typechecking Python files w/ mypy"
+	mypy src tests
+	@echo ""
+
+lint-travis:
+	@echo "--> Linting .travis.yml files"
+	travis lint --no-interactive
+	@echo ""
+
+lint-rst:
+	@echo "--> Linting .rst files"
+	rst-lint *.rst
+	@echo ""
+
+lint-doc:
+	@echo "--> Linting doc"
+	@echo "TODO"
+	#sphinx-build -W -b dummy docs/ docs/_build/
+	#sphinx-build -b dummy docs/ docs/_build/
+	@echo ""
+
+#
+# Formatting
+#
+format: format-py
+
+format-py:
+	docformatter -i -r src tests
+	black src tests
+	isort src tests
+
+#
+# Everything else
+#
+install:
+	poetry install
+
+doc: doc-html doc-pdf
+
+doc-html:
+	sphinx-build -W -b html docs/ docs/_build/html
+
+doc-pdf:
+	sphinx-build -W -b latex docs/ docs/_build/latex
+	make -C docs/_build/latex all-pdf
+
+clean:
+	rm -f **/*.pyc
+	find . -type d -empty -delete
+	rm -rf *.egg-info *.egg .coverage .eggs .cache .mypy_cache .pyre \
+		.pytest_cache .pytest .DS_Store  docs/_build docs/cache docs/tmp \
+		dist build pip-wheel-metadata junit-*.xml htmlcov coverage.xml
+
+tidy: clean
+	rm -rf .tox
+
+update-deps:
+	pip install -U pip setuptools wheel
+	poetry update
+
+publish: clean
+	git push --tags
+	poetry build
+	twine upload dist/*
+
+push:
+	rsync -e ssh -avz ./ pilaf:git/minij-proxy/
+
+fetch-results:
+	rsync -e ssh -avz pilaf:git/minij-proxy/results.csv results/
+

Procfile

+proxy: gunicorn -b localhost:3000 --pid server.pid \
+    -k uvicorn.workers.UvicornWorker -w 4 mynij_proxy:app

README.md

-Minij Proxy 
+Mynij Proxy 
 ===========
 
-Goal
-----
-
-Make a proxy that adds CORS headers to GET request for use by the Minij
+A proxy that adds CORS headers to GET request for use by the Mynij
 search engine.
 
-Current state
--------------
-
-1) Trying to figure out the best (best efficiency / elegance compromise) approach:
-
-    - Application: WSGI (sync) vs. ASGI (async)
-    
-       - WSGI: WebOb vs. Werkzeug (-> they are mostly interchangeable for what we need).
-    
-       - ASGI: Starlette, Blacksheep, Hug (-> Starlette is probably enough)
-    
-    - Client: Requests (sync) vs. httpx and aiohttp.client (async)
-      
-    - Server: Gunicorn, Uvicorn, Gunicorn+Uvicorn, uWsgi, Hypercorn...
-
-        See: https://docs.gunicorn.org/en/latest/design.html#choosing-a-worker-type
-
-2) Running benchmarks:
-
-    See: src/benchmarks.py. Some results are currently a bit surprising so this is a WIP.
-    
-    See also: https://gist.github.com/imbolc/15cab07811c32e7d50cc12f380f7f62f for a code example.
-
-    And: https://blog.miguelgrinberg.com/post/ignore-all-web-performance-benchmarks-including-this-one
-
-TODO
-----
-
-1) Finish benchmarks and choose an approach.
-
-    Actually we will probably go with:
-
-    - Starlette
-    - httpx (ou aiohttp.client)
-    - Gunicorn + uvicorn (`gunicorn -k uvicorn.workers.UvicornWorker`, see: <https://www.uvicorn.org/deployment/>)
-    - Do we need Nginx in front ? I don't think so but this remains an option.
-
-    But this may change after we complete de benchmarks.
+Uses Starlette as it's ASGI framework, and HTTPX as the HTTP client.
 
-2) Make a Buildout recipe for deployment on Rapid.Space.
+(We may switch to aiohttp.client instead of httpx later).
 
-3) Test and iterate
+Web server + workers are: gunicorn + uvicorn.

pyproject.toml

+[tool.poetry]
+name = "mynij-proxy"
+version = "0.1.0"
+description = ""
+authors = ["Stefane Fermigier <sf@fermigier.com>"]
+
+packages = [{ include = "*", from = "src" }]
+
+
+[tool.poetry.dependencies]
+python = "^3.9"
+
+httpx = "^0.18.1"
+gunicorn = "^20.1.0"
+uvicorn = "^0.13.4"
+uvloop = "^0.15.2"
+starlette = "^0.14.2"
+httptools = "*"
+
+
+[tool.poetry.dev-dependencies]
+black = "^21.4b2"
+isort = "^5.8.0"
+honcho = "*"
+pytest = "^6.2.4"
+requests = "^2.25.1"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

reference/with-proxy.txt

+HTTP/2 200 
+access-control-allow-origin: https://mynij.app.officejs.com
+cache-control: max-age=600, stale-while-revalidate=360000, stale-if-error=31536000, public
+content-type: text/html; charset=utf-8
+date: Mon, 03 May 2021 13:01:32 GMT
+last-modified: Mon, 03 May 2021 08:01:56 GMT
+server: Caddy
+server: Zope/(2.13.29, python 2.7.15, linux2) ZServer/1.1
+vary: Cookie,Authorization,Accept-Encoding
+

reference/without-proxy.txt

+HTTP/2 200 
+age: 63
+cache-control: max-age=600, stale-while-revalidate=360000, stale-if-error=31536000, public
+content-type: text/html; charset=utf-8
+date: Mon, 03 May 2021 13:01:52 GMT
+expires: Mon, 03 May 2021 13:11:52 GMT
+last-modified: Mon, 03 May 2021 08:01:56 GMT
+server: Caddy
+server: ATS/7.1.11
+vary: Cookie, Authorization, Accept-Encoding
+vary: Accept-Encoding
+x-cache-headers-set-by: CachingPolicyManager: /nexedi/caching_policy_manager
+

setup.cfg

+[isort]
+profile = black
+

src/mynij_proxy.py

+import traceback
+from typing import Mapping, Tuple
+
+import httpx
+from starlette.applications import Starlette
+from starlette.endpoints import HTTPEndpoint
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse, Response
+from starlette.routing import Route
+
+# Extremely aggressive and hardcoded value
+TIMEOUT = 10
+
+DEFAULT_ACCESS_URL = "https://mynij.app.officejs.com"
+
+httpx_session = httpx.AsyncClient()
+
+
+class ProxyEndPoint(HTTPEndpoint):
+    headers: Mapping[str, str]
+    url: str
+
+    async def get(self, request: Request):
+        self.headers = request.headers
+        self.url = request.query_params["url"]
+
+        status_code, content, new_headers = await self.fetch_content()
+        # debug(status_code, content, new_headers)
+        response = Response(content, status_code, new_headers)
+        return response
+
+    async def fetch_content(self) -> Tuple[int, bytes, dict]:
+        proxy_query_header = self.make_query_headers(self.headers)
+        body = b""
+        response_headers = {}
+
+        try:
+            proxy_response = await httpx_session.get(
+                self.url, headers=proxy_query_header, timeout=TIMEOUT
+            )
+            response_headers = self.filter_response_headers(proxy_response)
+            response_headers["Access-Control-Allow-Origin"] = self.get_access_url()
+            body = proxy_response.content
+            status = proxy_response.status_code
+
+        # except SSLError:
+        #     # Invalid SSL Certificate
+        #     status = 526
+        except TimeoutError:
+            traceback.print_exc()
+            # Gateway Timeout
+            status = 504
+        except httpx.TransportError:
+            traceback.print_exc()
+            # Service Unavailable
+            status = 503
+        except httpx.HTTPError:
+            traceback.print_exc()
+            # Internal Server Error
+            status = 500
+
+        return status, body, response_headers
+
+    def make_query_headers(self, headers: Mapping) -> Mapping:
+        request_headers = {}
+        HEADERS = [
+            "Content-Type",
+            "Accept",
+            "Accept-Language",
+            "Range",
+            "If-Modified-Since",
+            "If-None-Match",
+        ]
+        for k in HEADERS:
+            v = headers.get(k)
+            if v:
+                request_headers[k] = str(v)
+
+        return request_headers
+
+    def get_access_url(self):
+        return self.headers.get("Origin", DEFAULT_ACCESS_URL)
+
+    def filter_response_headers(self, proxy_response) -> dict[str, str]:
+        headers = {}
+        HEADERS = [
+            "Content-Disposition",
+            "Content-Type",
+            "Date",
+            "Last-Modified",
+            "Vary",
+            "Cache-Control",
+            "Etag",
+            "Accept-Ranges",
+            "Content-Range",
+        ]
+        for k, v in proxy_response.headers.items():
+            k = k.title()
+            if k in HEADERS:
+                headers[k] = v
+        return headers
+
+
+async def ping(request):
+    return PlainTextResponse("OK")
+
+
+routes = [
+    Route("/proxy", ProxyEndPoint),
+    Route("/ping", ping),
+]
+
+
+app = Starlette(debug=True, routes=routes)

tests/test_proxy.py

+from starlette.testclient import TestClient
+
+from mynij_proxy import app
+
+
+def test_app():
+    client = TestClient(app)
+    response = client.get("/ping")
+    assert response.status_code == 200