Use hook_url for http requests when available

5 files changed, 218 insertions, 60 deletions

diff --git a/slack/error.py b/slack/error.py
index 80af53d..917eddf 100644
--- a/slack/error.py
+++ b/slack/error.py
@@ -20,7 +20,7 @@ class HttpError(Exception):
         self,
         url: str,
         options: Dict[str, str],
-        return_code: int,
+        return_code: Optional[int],
         http_status_code: Optional[int],
         error: str,
     ):
diff --git a/slack/http.py b/slack/http.py
index b05cfb4..49b361e 100644
--- a/slack/http.py
+++ b/slack/http.py
@@ -9,7 +9,7 @@ import weechat
 
 from slack.error import HttpError
 from slack.log import LogLevel, log
-from slack.task import FutureProcess, sleep, weechat_task_cb
+from slack.task import FutureProcess, FutureUrl, sleep, weechat_task_cb
 from slack.util import get_callback_name
 
 
@@ -59,31 +59,69 @@ async def hook_process_hashtable(
     return command, return_code, out, err
 
 
-async def http_request(
-    url: str, options: Dict[str, str], timeout: int, max_retries: int = 5
-) -> str:
+async def hook_url(
+    url: str, options: Dict[str, str], timeout: int
+) -> Tuple[str, Dict[str, str], Dict[str, str]]:
+    future = FutureUrl()
+    weechat.hook_url(
+        url, options, timeout, get_callback_name(weechat_task_cb), future.id
+    )
+    return await future
+
+
+async def http_request_process(
+    url: str, options: Dict[str, str], timeout: int
+) -> Tuple[int, str, str]:
     options["header"] = "1"
     _, return_code, out, err = await hook_process_hashtable(
         f"url:{url}", options, timeout
     )
 
     if return_code != 0 or err:
+        raise HttpError(url, options, return_code, None, err)
+
+    parts = out.split("\r\n\r\nHTTP/")
+    headers, body = parts[-1].split("\r\n\r\n", 1)
+    http_status = int(headers.split(None, 2)[1])
+    return http_status, headers, body
+
+
+async def http_request_url(
+    url: str, options: Dict[str, str], timeout: int
+) -> Tuple[int, str, str]:
+    _, _, output = await hook_url(url, options, timeout)
+
+    if "error" in output:
+        raise HttpError(url, options, None, None, output["error"])
+
+    http_status = int(output["response_code"])
+    header_parts = output["headers"].split("\r\n\r\nHTTP/")
+    return http_status, header_parts[-1], output["output"]
+
+
+async def http_request(
+    url: str, options: Dict[str, str], timeout: int, max_retries: int = 5
+) -> str:
+    try:
+        if hasattr(weechat, "hook_url"):
+            http_status, headers, body = await http_request_url(url, options, timeout)
+        else:
+            http_status, headers, body = await http_request_process(
+                url, options, timeout
+            )
+    except HttpError as e:
         if max_retries > 0:
             log(
                 LogLevel.INFO,
                 f"HTTP error, retrying (max {max_retries} times): "
-                f"return_code: {return_code}, error: {err}, url: {url}",
+                f"return_code: {e.return_code}, error: {e.error}, url: {url}",
             )
             await sleep(1000)
             return await http_request(url, options, timeout, max_retries - 1)
-        raise HttpError(url, options, return_code, None, err)
-
-    parts = out.split("\r\n\r\nHTTP/")
-    last_header_part, body = parts[-1].split("\r\n\r\n", 1)
-    header_lines = last_header_part.split("\r\n")
-    http_status = int(header_lines[0].split(" ")[1])
+        raise
 
     if http_status == 429:
+        header_lines = headers.split("\r\n")
         for header in header_lines[1:]:
             name, value = header.split(":", 1)
             if name.lower() == "retry-after":
@@ -96,6 +134,6 @@ async def http_request(
                 return await http_request(url, options, timeout)
 
     if http_status >= 400:
-        raise HttpError(url, options, return_code, http_status, body)
+        raise HttpError(url, options, None, http_status, body)
 
     return body
diff --git a/slack/task.py b/slack/task.py
index 6dc02ce..d48af4a 100644
--- a/slack/task.py
+++ b/slack/task.py
@@ -6,6 +6,7 @@ from typing import (
     Awaitable,
     Callable,
     Coroutine,
+    Dict,
     Generator,
     List,
     Optional,
@@ -148,6 +149,10 @@ class FutureProcess(Future[Tuple[str, int, str, str]]):
     pass
 
 
+class FutureUrl(Future[Tuple[str, Dict[str, str], Dict[str, str]]]):
+    pass
+
+
 class FutureTimer(Future[Tuple[int]]):
     pass
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 1a0b4a0..42d2058 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -4,12 +4,12 @@ from unittest.mock import MagicMock, patch
 import pytest
 import weechat
 
-from slack.http import HttpError, http_request
-from slack.task import FutureProcess, FutureTimer, weechat_task_cb
+from slack.http import HttpError, http_request, http_request_process, http_request_url
+from slack.task import FutureProcess, FutureTimer, FutureUrl, weechat_task_cb
 from slack.util import get_callback_name
 
 
-@patch.object(weechat, "hook_process_hashtable")
+@patch.object(weechat, "hook_url")
 def test_http_request_success(mock_method: MagicMock):
     url = "http://example.com"
     options = {"option": "1"}
@@ -17,6 +17,33 @@ def test_http_request_success(mock_method: MagicMock):
     coroutine = http_request(url, options, timeout)
 
     future = coroutine.send(None)
+    assert isinstance(future, FutureUrl)
+
+    mock_method.assert_called_once_with(
+        url,
+        options,
+        timeout,
+        get_callback_name(weechat_task_cb),
+        future.id,
+    )
+
+    future.set_result(
+        (url, options, {"response_code": "200", "headers": "", "output": "response"})
+    )
+
+    with pytest.raises(StopIteration) as excinfo:
+        coroutine.send(None)
+    assert excinfo.value.value == "response"
+
+
+@patch.object(weechat, "hook_process_hashtable")
+def test_http_request_process_success(mock_method: MagicMock):
+    url = "http://example.com"
+    options = {"option": "1"}
+    timeout = 123
+    coroutine = http_request_process(url, options, timeout)
+
+    future = coroutine.send(None)
     assert isinstance(future, FutureProcess)
 
     mock_method.assert_called_once_with(
@@ -27,18 +54,34 @@ def test_http_request_success(mock_method: MagicMock):
         future.id,
     )
 
-    response = "response"
-    body = f"HTTP/2 200\r\n\r\n{response}"
+    body = f"HTTP/2 200\r\n\r\nresponse"
     future.set_result(("", 0, body, ""))
 
     with pytest.raises(StopIteration) as excinfo:
         coroutine.send(None)
-    assert excinfo.value.value == response
+    assert excinfo.value.value == (200, "HTTP/2 200", "response")
+
+
+def test_http_request_url_error():
+    url = "http://example.com"
+    coroutine = http_request_url(url, {}, 0)
+
+    future = coroutine.send(None)
+    assert isinstance(future, FutureUrl)
+    future.set_result((url, {}, {"error": "error"}))
+
+    with pytest.raises(HttpError) as excinfo:
+        coroutine.send(None)
+
+    assert excinfo.value.url == url
+    assert excinfo.value.return_code == None
+    assert excinfo.value.http_status_code == None
+    assert excinfo.value.error == "error"
 
 
-def test_http_request_error_process_return_code():
+def test_http_request_process_error_return_code():
     url = "http://example.com"
-    coroutine = http_request(url, {}, 0, max_retries=0)
+    coroutine = http_request_process(url, {}, 0)
 
     future = coroutine.send(None)
     assert isinstance(future, FutureProcess)
@@ -53,9 +96,9 @@ def test_http_request_error_process_return_code():
     assert excinfo.value.error == ""
 
 
-def test_http_request_error_process_stderr():
+def test_http_request_process_error_stderr():
     url = "http://example.com"
-    coroutine = http_request(url, {}, 0, max_retries=0)
+    coroutine = http_request_process(url, {}, 0)
 
     future = coroutine.send(None)
     assert isinstance(future, FutureProcess)
@@ -70,24 +113,24 @@ def test_http_request_error_process_stderr():
     assert excinfo.value.error == "err"
 
 
-def test_http_request_error_process_http():
+def test_http_request_error_http_status():
     url = "http://example.com"
-    coroutine = http_request(url, {}, 0, max_retries=0)
+    coroutine = http_request(url, {}, 0)
 
     future = coroutine.send(None)
-    assert isinstance(future, FutureProcess)
+    assert isinstance(future, FutureUrl)
 
-    response = "response"
-    body = f"HTTP/2 400\r\n\r\n{response}"
-    future.set_result(("", 0, body, ""))
+    future.set_result(
+        (url, {}, {"response_code": "400", "headers": "", "output": "response"})
+    )
 
     with pytest.raises(HttpError) as excinfo:
         coroutine.send(None)
 
     assert excinfo.value.url == url
-    assert excinfo.value.return_code == 0
+    assert excinfo.value.return_code == None
     assert excinfo.value.http_status_code == 400
-    assert excinfo.value.error == response
+    assert excinfo.value.error == "response"
 
 
 def test_http_request_error_retry_success():
@@ -95,23 +138,23 @@ def test_http_request_error_retry_success():
     coroutine = http_request(url, {}, 0, max_retries=2)
 
     future_1 = coroutine.send(None)
-    assert isinstance(future_1, FutureProcess)
-    future_1.set_result(("", -2, "", ""))
+    assert isinstance(future_1, FutureUrl)
+    future_1.set_result((url, {}, {"error": "error"}))
 
     future_2 = coroutine.send(None)
     assert isinstance(future_2, FutureTimer)
     future_2.set_result((0,))
 
     future_3 = coroutine.send(None)
-    assert isinstance(future_3, FutureProcess)
+    assert isinstance(future_3, FutureUrl)
 
-    response = "response"
-    body = f"HTTP/2 200\r\n\r\n{response}"
-    future_3.set_result(("", 0, body, ""))
+    future_3.set_result(
+        (url, {}, {"response_code": "200", "headers": "", "output": "response"})
+    )
 
     with pytest.raises(StopIteration) as excinfo:
         coroutine.send(None)
-    assert excinfo.value.value == response
+    assert excinfo.value.value == "response"
 
 
 def test_http_request_error_retry_error():
@@ -119,41 +162,71 @@ def test_http_request_error_retry_error():
     coroutine = http_request(url, {}, 0, max_retries=2)
 
     future_1 = coroutine.send(None)
-    assert isinstance(future_1, FutureProcess)
-    future_1.set_result(("", -2, "", ""))
+    assert isinstance(future_1, FutureUrl)
+    future_1.set_result((url, {}, {"error": "error"}))
 
     future_2 = coroutine.send(None)
     assert isinstance(future_2, FutureTimer)
     future_2.set_result((0,))
 
     future_3 = coroutine.send(None)
-    assert isinstance(future_3, FutureProcess)
-    future_3.set_result(("", -2, "", ""))
+    assert isinstance(future_3, FutureUrl)
+    future_3.set_result((url, {}, {"error": "error"}))
 
     future_4 = coroutine.send(None)
     assert isinstance(future_4, FutureTimer)
     future_4.set_result((0,))
 
     future_5 = coroutine.send(None)
-    assert isinstance(future_5, FutureProcess)
-    future_5.set_result(("", -2, "", ""))
+    assert isinstance(future_5, FutureUrl)
+    future_5.set_result((url, {}, {"error": "error"}))
 
     with pytest.raises(HttpError) as excinfo:
         coroutine.send(None)
 
     assert excinfo.value.url == url
-    assert excinfo.value.return_code == -2
+    assert excinfo.value.return_code == None
     assert excinfo.value.http_status_code == None
-    assert excinfo.value.error == ""
+    assert excinfo.value.error == "error"
 
 
-def test_http_request_multiple_headers():
+def test_http_request_url_multiple_headers():
     url = "http://example.com"
-    coroutine = http_request(url, {}, 0)
+    coroutine = http_request_url(url, {}, 0)
+    future = coroutine.send(None)
+    assert isinstance(future, FutureUrl)
+
+    headers = (
+        dedent(
+            f"""
+            HTTP/1.1 200 Connection established
+
+            HTTP/2 200
+            content-type: application/json; charset=utf-8
+            """
+        )
+        .strip()
+        .replace("\n", "\r\n")
+    )
+    future.set_result(
+        (url, {}, {"response_code": "200", "headers": headers, "output": "response"})
+    )
+
+    with pytest.raises(StopIteration) as excinfo:
+        coroutine.send(future)
+    assert excinfo.value.value == (
+        200,
+        "2 200\r\ncontent-type: application/json; charset=utf-8",
+        "response",
+    )
+
+
+def test_http_request_process_multiple_headers():
+    url = "http://example.com"
+    coroutine = http_request_process(url, {}, 0)
     future = coroutine.send(None)
     assert isinstance(future, FutureProcess)
 
-    response = "response"
     body = (
         dedent(
             f"""
@@ -162,7 +235,7 @@ def test_http_request_multiple_headers():
             HTTP/2 200
             content-type: application/json; charset=utf-8
 
-            {response}
+            response
             """
         )
         .strip()
@@ -172,7 +245,11 @@ def test_http_request_multiple_headers():
 
     with pytest.raises(StopIteration) as excinfo:
         coroutine.send(future)
-    assert excinfo.value.value == response
+    assert excinfo.value.value == (
+        200,
+        "2 200\r\ncontent-type: application/json; charset=utf-8",
+        "response",
+    )
 
 
 @patch.object(weechat, "hook_timer")
@@ -181,10 +258,19 @@ def test_http_request_ratelimit(mock_method: MagicMock):
     coroutine = http_request(url, {}, 0)
 
     future_1 = coroutine.send(None)
-    assert isinstance(future_1, FutureProcess)
-
-    body = "HTTP/2 429\r\nRetry-After: 12\r\n\r\n"
-    future_1.set_result(("", 0, body, ""))
+    assert isinstance(future_1, FutureUrl)
+
+    future_1.set_result(
+        (
+            url,
+            {},
+            {
+                "response_code": "429",
+                "headers": "HTTP/2 429\r\nRetry-After: 12",
+                "output": "response",
+            },
+        )
+    )
 
     future_2 = coroutine.send(None)
     assert isinstance(future_2, FutureTimer)
@@ -195,11 +281,12 @@ def test_http_request_ratelimit(mock_method: MagicMock):
     )
 
     future_3 = coroutine.send(None)
-    assert isinstance(future_3, FutureProcess)
+    assert isinstance(future_3, FutureUrl)
 
-    response = "response"
-    future_3.set_result(("", 0, f"HTTP/2 200\r\n\r\n{response}", ""))
+    future_3.set_result(
+        (url, {}, {"response_code": "200", "headers": "", "output": "response"})
+    )
 
     with pytest.raises(StopIteration) as excinfo:
         coroutine.send(None)
-    assert excinfo.value.value == response
+    assert excinfo.value.value == "response"
diff --git a/typings/weechat.pyi b/typings/weechat.pyi
index 49191fa..db2b5ce 100644
--- a/typings/weechat.pyi
+++ b/typings/weechat.pyi
@@ -1362,6 +1362,32 @@ def hook_process_hashtable(command: str, options: Dict[str, str], timeout: int,
     ...
 
 
+def hook_url(url: str, options: Dict[str, str], timeout: int, callback: str, callback_data: str) -> str:
+    """`hook_url in WeeChat plugin API reference <https://weechat.org/doc/weechat/api/#_hook_url>`_
+    ::
+
+        # example
+        def my_url_cb(data: str, url: str, options: Dict[str, str], output: Dict[str, str]) -> int:
+            weechat.prnt("", "output: %s" % output)
+            return weechat.WEECHAT_RC_OK
+
+        # example 1: output to a file
+        hook1 = weechat.hook_url("https://weechat.org/",
+                                 {"file_out": "/tmp/weechat.org.html"},
+                                 20000, "my_url_cb", "")
+
+        # example 2: custom HTTP headers, output sent to callback
+        options = {
+            "httpheader": "\n".join([
+                "Header1: value1",
+                "Header2: value2",
+            ]),
+        }
+        hook2 = weechat.hook_url("http://localhost:8080/", options, 20000, "my_url_cb", "")
+    """
+    ...
+
+
 def hook_connect(proxy: str, address: str, port: int, ipv6: int, retry: int, local_hostname: str,
                  callback: str, callback_data: str) -> str:
     """`hook_connect in WeeChat plugin API reference <https://weechat.org/doc/weechat/api/#_hook_connect>`_
@@ -2661,7 +2687,9 @@ def hdata_compare(hdata: str, pointer1: str, pointer2: str, name: str, case_sens
         hdata = weechat.hdata_get("buffer")
         buffer1 = weechat.buffer_search("irc", "libera.#weechat")
         buffer2 = weechat.buffer_search("irc", "libera.#weechat-fr")
-        weechat.prnt("", "number comparison = %d" % weechat.hdata_compare(hdata, buffer1, buffer2, "number", 0))
+        weechat.prnt("", "comparison of buffer number = %d" % weechat.hdata_compare(hdata, buffer1, buffer2, "number", 0))
+        weechat.prnt("", "comparison of number of lines = %d" % weechat.hdata_compare(hdata, buffer1, buffer2, "own_lines.lines_count", 0))
+        weechat.prnt("", "comparison of local variable = %d" % weechat.hdata_compare(hdata, buffer1, buffer2, "local_variables.myvar", 0))
     """
     ...