diff --git a/.gitignore b/.gitignore index 4254f76..b1c74ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ .DS_Store dist picohttp.egg-info -pyproject.toml diff --git a/README.md b/README.md index db5412f..58594aa 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The HttpServer class is used to instantiate an HTTP server object. **`class HttpServer(port=80, handler=staticResource, args=(), threads=True, start=True, block=True)`** port - The port to listen on. + The port to listen on. An int or list of int values. handler The function defined to be the request handler. @@ -28,14 +28,23 @@ The HttpServer class is used to instantiate an HTTP server object. threads If True handle each request in a separate thread, otherwise process them sequentially. + reuse + If True allow the reuse of the server address, otherwise the server will not start if the address is already in use. + start If True immediately start the server, otherwise the start() function must be called. block If True the server will block indefinitely when it is started, otherwise the function starting it will return. - start() - Start the server if it was instantiated with start=False. + one_request + If True, shutdown the server after processing one request. + + accept_period + The time in seconds to wait for a connection before checking if the server should be shutdown. + + accept_queue_size + The maximum number of connections to queue before refusing new connections. When a request is received it is parsed by the HTTP server and an HttpRequest object containing the request elements is passed to the request handler. @@ -105,6 +114,8 @@ A helper function that facilitates the serving of static resources my be called The HTTP mime type will be placed into a Content-Type header based on the file extension of the resource. This is the default that should be used if it can't be determined. ### Examples + + This is a minimal HTTP server. It listens on port 80 and resources contained in the directory where the application is run from. ``` import picohttp diff --git a/picohttp/httpServer.py b/picohttp/httpServer.py index 1bbf3f3..21b3f22 100644 --- a/picohttp/httpServer.py +++ b/picohttp/httpServer.py @@ -8,21 +8,63 @@ from rutifu import * from .httpClasses import * from .staticResource import * +from typing import Union + class HttpServer(object): - def __init__(self, port=80, handler=staticResource, args=(), threads=True, reuse=True, block=True, start=True): + def __init__(self, port: Union[int, list] = 80, handler=staticResource, args=(), threads=True, reuse=True, + block=True, start=True, one_request=False, accept_period: float = 3.0, accept_queue_size: int = 5): + """ + :param port: int or list of ints - ports to try to use. First successful bind() is utilized. + :param handler: function to handle requests. The handler should take a request and response object as arguments. + AND if the handler returns True AND one_request, the server's shutdown flag will be set to True. + :param args: Optional tuple of handler specific arguments. Last member in self.args is the server object. So the + handler can access the server object if needed to set shutdown = True + :param threads: True -> each request is handled in a separate thread. There is always a dedicated thread created + to listen for incoming connects. If not threads, the listener thread will also handle the requests. + :param reuse: If True, the server will set the SO_REUSEADDR socket option. to get around TIME_WAIT issues. + :param block: True -> server will block indefinitely waiting for incoming connections. False -> the invoker must + use its own strategy to block (e.g. sleep(n) or wait for a signal), settings the shutdown flag to True. + :param start: True -> server will start immediately. False -> server will not start until the start() method is + invoked. + :param one_request: True -> server will shut down after processing one request. Regardless of the return value + of the handler. If false, then the handler's return value of True will set the server's shutdown flag to True. + :param accept_period: time to wait for a connection (via accept) before checking if we should shut down. + :param accept_queue_size: size of the accept queue for incoming connections. Make this number large enough to + reduce the likelihood that an incoming connection that may be used to terminate the server is blocked waiting + 'too' long. + """ self.ports = listize(port) self.port = 0 self.handler = handler - self.args = args + self.args = (*args, self) # TODO ? make the HttpServer ref the first or last in the arg tuple? as the expansion + # of this tuple when invoking the handler provides positional args to the handler rather than named args. Which + # is a strong argument for making the server object the first arg in the tuple. But most handlers won't need it + # which makes a strong argument for making it the last arg in the tuple. And this is consistent with my + # understanding of the usage of this lean, mean HttpServer. Simple, easy to use. self.threads = threads self.reuse = reuse self.block = block self.socket = None + self.one_request_only = one_request # we shut down after one request is processed. + self.accept_period = accept_period + self.accept_queue_size = accept_queue_size # in blocking mode, if an incoming connection is used to trigger a + # shutdown, we need to ensure that the accept() call is not blocked waiting for that connection. Definitely an + # issue when using tightly timed clients in tests. Note that I'm not bothering to use a mutex (threading.Lock) + # to protect the shutdown flag. This is because the shutdown flag is only set to True and never reset to False. + # simple semaphore usage, IMHO. + self.shutdown = False if start: self.start() def start(self): + """ + Start the server by opening a socket on the specified port and listening for incoming connections. The + listening is either in a blocking mode or non-blocking. If non-blocking, the invoker must block using its + own strategy (e.g. sleep(n) or wait for a signal). + :return: 0 if no port was successfully used to listen on (bind failed for all our ports), else the port number + that is being listened on. (useful to convey to clients as this is the port to be used). + """ debug("debugHttpServer", "httpServer", "starting") self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.reuse: @@ -37,10 +79,12 @@ def start(self): except OSError: pass if self.port: - self.socket.listen(1) - startThread("httpserver", self.getRequests) + self.socket.listen(self.accept_queue_size) + startThread(f"httpserver_{self.port}", self.getRequests) + # at this point, we now have another thread that is listening for incoming connections with a request + # so we might need to block (if not, our caller will need to block using its own strategy) if self.block: - block() + self._block() return self.port else: self.socket.close() @@ -50,21 +94,40 @@ def start(self): # wait for requests def getRequests(self): debug("debugHttpServer", "waiting for request") + # we might be notified to shut down after we start waiting so use the accept_period max wait time + self.socket.settimeout(self.accept_period) # never block forever if there is a possible shutdown needed while True: - (client, addr) = self.socket.accept() - if self.threads: - startThread("httpserver_"+str(addr[0])+"_"+str(addr[1]), self.handleConnection, args=(client, addr,)) - else: - self.handleConnection(client, addr) + # wait for a connection unless we've already been set to shut down + if self.shutdown: + self.socket.close() + return + try: + (client, addr) = self.socket.accept() + if self.threads: + startThread("httpserver_"+str(addr[0])+"_"+str(addr[1]), self.handleConnection, + args=(client, addr,)) + else: + self.handleConnection(client, addr) + if self.one_request_only: + self.shutdown = True + self.socket.close() + return # our thread runs-down after one request + except socket.timeout: + if self.shutdown: + self.socket.close() + return + # otherwise we just keep looping, waiting for a connection. def handleConnection(self, client, addr): + stop = False request = HttpRequest() self.parseRequest(client, addr, request) debugRequest("debugHttpServer", addr, request) # send it to the request handler response = HttpResponse("HTTP/1.0", 200, {}, None) try: - self.handler(request, response, *self.args) + if self.handler(request, response, *self.args): + stop = True except Exception as ex: logException("exception in request handler", ex) response.status = 500 @@ -72,6 +135,8 @@ def handleConnection(self, client, addr): self.sendResponse(client, addr, response) debugResponse("debugHttpServer", addr, response) client.close() + if stop and self.one_request_only: + self.shutdown = True def parseRequest(self, client, addr, request): clientFile = client.makefile() @@ -121,3 +186,13 @@ def sendResponse(self, client, addr, response): except BrokenPipeError: # can't do anything about this log("sendResponse", "broken pipe", addr[0]) return + + def _block(self): + """ + Block the calling thread indefinitely until we are set to shut down + :return: None + """ + while True: + if self.shutdown: + return + time.sleep(1) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..cac3f2e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "packaging>=24.2" # see https://github.com/pypa/twine/issues/1216 for why (and this is not sufficient to overcome the issue) + # see this issue as well: https://github.com/pypa/setuptools/issues/4759 +] +build-backend = "setuptools.build_meta" + +[project] +name="picohttp" +version="0.5.1" +requires-python= ">= 3.7" +authors = [ + { name= "Joe Deuhl", email="joe@thebuehls.com" }, + { name= "Steve Hespelt", email= "shespelt@stevens.edu"}, +] +description = "An HTTP 1.0 server framework that is useful for supporting applications where the behavior is constrained and relatively simple" +dependencies = [ + "rutifu" +] +readme = "README.md" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] +# license-files key added in Dec 2024, not recognized by twine validation - why the workarounds (so I can publish this) +# license-files = ["LICENSE"] +#license.file = 'LICENSE' # "GPL-3.0-only" + +[project.optional-dependencies] +# stuff that can be included via pkg-main[option1] + +[project.scripts] +# python script that can get wrapper applied, so script invoked as command + +[project.urls] +Repository="https://github.com/jbuehl/picohttp" + +[tool.setuptools] +license-files = [] # see https://github.com/pypa/twine/issues/1216, https://github.com/pypa/setuptools/issues/4759 for why + # upgrading packaging to the latest (24.2) does not fix the issue + +[tool.setuptools.packages.find] +where = ["./"] +include = ["picohttp"] +exclude = ["./tests**"] + + +[tool.poetry] + +[tool.mypy] +mypy_path = 'picohttp/' + +[tool.coverage.run] +source = [ + "picohttp/**" +] +omit = [ + "tests/skip*.py" +] diff --git a/tests/test_blocking.py b/tests/test_blocking.py new file mode 100644 index 0000000..00db870 --- /dev/null +++ b/tests/test_blocking.py @@ -0,0 +1,151 @@ +import requests +import threading +import unittest + +import time + +import urllib3.exceptions + +from picohttp.httpServer import HttpServer as PicoHttpServer +from picohttp.httpClasses import HttpRequest as PicoHttpRequest +from picohttp.httpClasses import HttpResponse as PicoHttpResponse + +from .test_nonblocking import (no_op_listener, find_our_thread, http_client) + + +class ClientThread(threading.Thread): + def __init__(self, port: int, test_obj=None, server: PicoHttpServer = None, + delay: float = 5.0, shutdown: bool = False): + super().__init__(None) + self.port = port + self.test_obj: BlockingTestCase = test_obj + self.start_delay = delay + self.server = server + self.shutdown = shutdown + + def run(self): + # since our test is using a blocking server, we use a delay to give the server time to start + time.sleep(self.start_delay) + try: + print("ClientThread http_client attempting to connect to port: ", self.port, ' after delay of: ', + self.start_delay) + sc: int = http_client(self.port) + if sc < 300: + self.test_obj.increment_client_count() + print("ClientThread http_client returned a SC < 300: ", sc) + else: + print("UhOH - our ClientThread http_client did not return a SC < 300: ", sc) + except urllib3.exceptions.MaxRetryError: + self.shutdown = True # need to shut down the server if we can't connect + if self.shutdown and self.server is not None: + self.server.shutdown = True + + +class DelayShutdownThread(threading.Thread): + def __init__(self, server: PicoHttpServer, delay: float = 5.0): + super().__init__(None) + self.server = server + self.delay = delay + + def run(self): + time.sleep(self.delay) + self.server.shutdown = True + + +class BlockingTestCase(unittest.TestCase): + the_ports = [8080, 8090, 8180, 8190] # have each test use a different port + test_num = 0 + + def setUp(self): + self.ports_to_try = [BlockingTestCase.the_ports[BlockingTestCase.test_num]] # [8080, 8090, 8180] + BlockingTestCase.test_num += 1 + if BlockingTestCase.test_num >= len(BlockingTestCase.the_ports): + BlockingTestCase.test_num = 0 # reset in case we have more tests than ports + self.num_clients = 0 + self.counter_lock: threading.Lock = threading.Lock() + + def increment_client_count(self): + self.counter_lock.acquire() + self.num_clients += 1 + self.counter_lock.release() + + def test_blocking_no_client(self): + """ + Test the PicoHttpServer with a short period for accepting incoming connections. This is a non-blocking test. + NOTE that there is no client attempt to connect. This is by design for this test as we want to confirm + that the server thread will exit after the specified period without a connection. + :return: + """ + shutdown_delay = 4.0 + start_time = time.time() + server = PicoHttpServer(self.ports_to_try, no_op_listener, threads=False, block=True, start=False, + one_request=False, accept_period=3) + shutter = DelayShutdownThread(server, shutdown_delay) + shutter.start() + server.start() + end_time = time.time() + # time for our accept thread to run-down (if it hasn't already) - give accept_period + 1 at a minimum + time.sleep(4) + thr_name = find_our_thread('httpserver_', server.port) + self.assertIsNone(thr_name) + self.assertLess(end_time - start_time, shutdown_delay + 1.0) # allow for overhead + + def test_blocking_one_client(self): + """ + Test the PicoHttpServer with a short period for accepting incoming connections (how long to block before + checking if the server shutdown flag. + NOTE that there is only one client attempt to connect. This is by design for this test as we want to confirm + that the one_request==True will cause the server to shut down after the client request is processed. That's + why the ClientThread instance's shutdown flag == False, not its job to shut down the server (in this test). + :return: + """ + run_delay: float = 4.0 + start_time = time.time() + server = PicoHttpServer(self.ports_to_try, no_op_listener, threads=False, block=True, start=False, + one_request=True, accept_period=3) + # since we can't start the server until after we construct the client, assume the 1st port is available :-( + the_client = ClientThread(self.ports_to_try[0], self, server, delay=run_delay, shutdown=False) + the_client.start() + # it's blocking so we depend on the one_request flag to shut down the server (we need either a request to + # occur OR something to set the server's shutdown flag to True). In this test, we have a client making a request + server.start() + end_time = time.time() + time.sleep(4) # wait at least the accept_period + 1.0 + self.assertFalse( the_client.is_alive(), 'Client thread is still alive') + thr_name = find_our_thread('httpserver_', server.port) + self.assertIsNone(thr_name) + self.assertEqual(self.num_clients, 1) + self.assertLess(end_time - start_time, run_delay + 2.0) # allow for overhead + + def test_blocking_threading_clients(self): + """ + Test the PicoHttpServer with a short period for accepting incoming connections. This is a blocking test. + NOTE that there is are several clients attempting to connect. The client is the longer start delay is configured + to try to shut down the associated server. + + :return: + """ + run_delay: float = 5.0 + start_time = time.time() + server = PicoHttpServer(self.ports_to_try, no_op_listener, threads=True, block=True, start=False, + one_request=False, accept_period=3) + # since we can't start the server until after we construct the client, assume the 1st port is available :-( + c1 = ClientThread(self.ports_to_try[0], self, server, delay=run_delay, shutdown=False) + c2 = ClientThread(self.ports_to_try[0], self, server, delay=run_delay+2.0, shutdown=False) + delay_shutdown = DelayShutdownThread(server, run_delay * 3.0) + delay_shutdown.start() + c1.start() + c2.start() + server.start() # its blocking so we wait for the client cause the shutdown [as constructed] + end_time = time.time() + time.sleep(4.0) # time for our accept thread to run-down - wait at least accept_period + 1 + thr_name = find_our_thread('httpserver_', server.port) + self.assertIsNone(thr_name) + self.assertEqual(self.num_clients, 2) + self.assertFalse(c1.is_alive(), "Client 1 thread is still alive") + self.assertFalse(c2.is_alive(), "Client 2 thread is still alive") + self.assertLess(end_time - start_time, (run_delay * 3.0) + 1.0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_nonblocking.py b/tests/test_nonblocking.py new file mode 100644 index 0000000..55f6faf --- /dev/null +++ b/tests/test_nonblocking.py @@ -0,0 +1,163 @@ +import requests +import threading +import unittest + +import time +from typing import Union + +from picohttp.httpServer import HttpServer as PicoHttpServer +from picohttp.httpClasses import HttpRequest as PiceHttpRequest +from picohttp.httpClasses import HttpResponse as PiceHttpResponse + +CLIENT_TIMEOUT= 5.0 # plenty of time to start getting a response from the server + + +def no_op_listener(req: PiceHttpRequest, res: PiceHttpResponse, args: tuple = ()) -> bool: + return False # we do NOT want to initiate server shutdown + + +def find_our_thread(our_name_prefix: str, the_port: int = 0) -> Union[str,None]: + """ + :param our_name_prefix: the "httpserver_ prefix is used in every thread name created by the PicoHttpServer + :param the_port: if 0, we are looking for any thread with the prefix. If not 0, we are looking for a specific thread. + :return: + """ + current_threads:[] = threading.enumerate() + for t in current_threads: + if the_port > 0 and t.name == f'{our_name_prefix}{the_port}' : + return t.name + elif the_port == 0 and t.name.startswith(our_name_prefix): + return t.name + return None + + +class ClientThread(threading.Thread): + def __init__(self, port: int, test_obj = None): + super().__init__(None) + self.port = port + self.test_obj: MyTestCase = test_obj + + def run(self): + sc: int = http_client(self.port ) + if sc < 300: + self.test_obj.increment_client_count() + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.ports_to_try = [8080, 8090, 8180] + self.num_clients = 0 + self.counter_lock: threading.Lock = threading.Lock() + + def increment_client_count(self): + self.counter_lock.acquire() + self.num_clients += 1 + self.counter_lock.release() + + def test_nonblocking_no_client(self): + """ + Test the PicoHttpServer with a short period for accepting incoming connections. This is a non-blocking test. + NOTE that there is no client attempt to connect. This is by design for this test as we want to confirm + that the server thread will exit after the specified period without a connection. + :return: + """ + n: int = 0 + maxWaits: int = 5 + wait_period = 5.0 + start_time = time.time() + server = PicoHttpServer(self.ports_to_try, no_op_listener, threads=False, block=False, start=True, + one_request=False, accept_period=4) + while n < maxWaits: + if n ==3: + server.shutdown = True + time.sleep(wait_period) + # at this point, the thread(s) used by the PicoHttpServer should have exited. Need to verify this. TODO + n += 1 + end_time = time.time() + thr_name = find_our_thread('httpserver_', server.port) + self.assertIsNone(thr_name) + self.assertLess(end_time - start_time, (maxWaits*wait_period) +1.0) # looping 5 times with a 5 second sleep each time. allow for overhead + + def test_nonblocking_one_client(self): + """ + Test the PicoHttpServer with a short period for accepting incoming connections. This is a non-blocking test. + NOTE that there is no client attempt to connect. This is by design for this test as we want to confirm + that the server thread will exit after the specified period without a connection. + :return: + """ + sc: int = 501 + n: int = 0 + maxWaits: int = 5 + wait_period = 5.0 + start_time = time.time() + server = PicoHttpServer(self.ports_to_try, no_op_listener, threads=False, block=False, start=True, + one_request=False, accept_period=4) + while n < maxWaits: + if n == 1: + sc = http_client(server.port ) + if n ==3: + server.shutdown = True + time.sleep(wait_period) + # at this point, the thread(s) used by the PicoHttpServer should have exited. Need to verify this. TODO + n += 1 + + end_time = time.time() + thr_name = find_our_thread('httpserver_', server.port) + self.assertIsNone(thr_name) + self.assertLess(sc, 300) + self.assertLess(end_time - start_time, (maxWaits*wait_period)+1.0) # looping 5 times with a 5 second sleep each time. allow for overhead + + def test_nonblocking_threading_clients(self): + """ + Test the PicoHttpServer with a short period for accepting incoming connections. This is a non-blocking test. + We launch multiple clients on their own thread. + :return: None + """ + sc: int = 501 + n: int = 0 + maxWaits: int = 5 + wait_period = 5.0 + start_time = time.time() + server = PicoHttpServer(self.ports_to_try, no_op_listener, threads=True, block=False, start=True, + one_request=False, accept_period=4) + c1 = None + c2 = None + while n < maxWaits: + if n == 1: # start 2 clients + c1 = ClientThread(server.port, self) + c2 = ClientThread(server.port, self) + c1.start() + c2.start() + if n ==3: + server.shutdown = True + time.sleep(wait_period) # make this period at least 1 second longer than the accept_period + # at this point, the thread(s) used by the PicoHttpServer should have exited. We verify this below + n += 1 + end_time = time.time() + thr_name = find_our_thread('httpserver_', server.port) # accept port should have run down by now. + self.assertFalse( c1.is_alive(), "Client 1 thread is still alive") + self.assertFalse( c2.is_alive(), "Client 2 thread is still alive") + self.assertIsNone(thr_name) + self.assertEqual( self.num_clients, 2) # did all the clients connect? and the responses were ok? + # we should have exited the server no later than the max loop time + 1 second for overhead + self.assertLess(end_time - start_time, (maxWaits * wait_period)+1.0, 'Server sleep loop took too long' ) + + +def http_client( port: int, host: str = 'localhost' ) -> int: + """ + Create an HTTP client connecting to the server under test. + :param port: the port the server is listening on + :param host: usually localhost is appropriate for our tests + :return: + """ + response = None + try: + response = requests.get(f'http://{host}:{port}', timeout=CLIENT_TIMEOUT) + except requests.exceptions.Timeout as e: + pass # just eat it as we are testing the server not the client, 501 SC is assumed + return response.status_code if response is not None else 501 + + + +if __name__ == '__main__': + unittest.main()