feat: add `max_payload_size` parameter to `path_deploy()` and some of `start_server()`

pywebio/input.py

@@ -67,6 +67,7 @@ from collections.abc import Mapping
 from .io_ctrl import single_input, input_control, output_register_callback
 from .session import get_current_session, get_current_task_id
 from .utils import Setter, is_html_safe_value, parse_file_size
+from .platform import utils as platform_setting
 
 logger = logging.getLogger(__name__)
 
@@ -538,13 +539,21 @@ def file_upload(label='', accept=None, name=None, placeholder='Choose file', mul
 
     .. note::
     
-        If uploading large files, please pay attention to the file upload size limit setting of the web framework. When using :func:`start_server <pywebio.platform.start_server>` to start the PyWebIO application, the maximum file size to be uploaded allowed by the web framework can be set through the `websocket_max_message_size` parameter
+        If uploading large files, please pay attention to the file upload size limit setting of the web framework.
+        When using :func:`start_server() <pywebio.platform.tornado.start_server>`/:func:`path_deploy() <pywebio.platform.path_deploy>` to start the PyWebIO application,
+        the maximum file size to be uploaded allowed by the web framework can be set through the ``max_payload_size`` parameter.
 
     """
     item_spec, valid_func = _parse_args(locals())
     item_spec['type'] = 'file'
-    item_spec['max_size'] = parse_file_size(max_size)
-    item_spec['max_total_size'] = parse_file_size(max_total_size)
+    item_spec['max_size'] = parse_file_size(max_size) or platform_setting.MAX_PAYLOAD_SIZE
+    item_spec['max_total_size'] = parse_file_size(max_total_size) or platform_setting.MAX_PAYLOAD_SIZE
+
+    if platform_setting.MAX_PAYLOAD_SIZE:
+        if item_spec['max_size'] > platform_setting.MAX_PAYLOAD_SIZE or \
+                item_spec['max_total_size'] > platform_setting.MAX_PAYLOAD_SIZE:
+            raise ValueError('The `max_size` and `max_total_size` value can not exceed the backend payload size limit. '
+                             'Please increase the `max_total_size` of `start_server()`/`path_deploy()`')
 
     def read_file(data):
         if not multiple:

pywebio/platform/django.py

@@ -5,9 +5,10 @@ import threading
 
 from django.http import HttpResponse, HttpRequest
 
+from . import utils
 from .httpbased import HttpContext, HttpHandler, run_event_loop
 from .utils import make_applications, cdn_validation
-from ..utils import STATIC_PATH, iscoroutinefunction, isgeneratorfunction, get_free_port
+from ..utils import STATIC_PATH, iscoroutinefunction, isgeneratorfunction, get_free_port, parse_file_size
 
 logger = logging.getLogger(__name__)
 
@@ -101,7 +102,7 @@ def start_server(applications, port=8080, host='', cdn=True, static_dir=None,
                  allowed_origins=None, check_origin=None,
                  session_expire_seconds=None,
                  session_cleanup_interval=None,
-                 debug=False, **django_options):
+                 debug=False, max_payload_size='200M', **django_options):
     """Start a Django server to provide the PyWebIO application as a web service.
 
     :param bool debug: Django debug mode.
@@ -128,11 +129,15 @@ def start_server(applications, port=8080, host='', cdn=True, static_dir=None,
 
     cdn = cdn_validation(cdn, 'warn')
 
+    max_payload_size = parse_file_size(max_payload_size)
+    utils.MAX_PAYLOAD_SIZE = max_payload_size
+
     django_options.update(dict(
         DEBUG=debug,
         ALLOWED_HOSTS=["*"],  # Disable host header validation
         ROOT_URLCONF=__name__,  # Make this module the urlconf
         SECRET_KEY=get_random_string(10),  # We aren't using any security features but Django requires this setting
+        DATA_UPLOAD_MAX_MEMORY_SIZE=max_payload_size
     ))
     django_options.setdefault('LOGGING', {
         'version': 1,
@@ -177,7 +182,7 @@ def start_server(applications, port=8080, host='', cdn=True, static_dir=None,
     if use_tornado_wsgi:
         import tornado.wsgi
         container = tornado.wsgi.WSGIContainer(app)
-        http_server = tornado.httpserver.HTTPServer(container)
+        http_server = tornado.httpserver.HTTPServer(container, max_buffer_size=max_payload_size)
         http_server.listen(port, address=host)
         tornado.ioloop.IOLoop.current().start()
     else:

pywebio/platform/flask.py

@@ -7,10 +7,11 @@ import threading
 
 from flask import Flask, request, send_from_directory, Response
 
+from . import utils
 from .httpbased import HttpContext, HttpHandler, run_event_loop
 from .utils import make_applications, cdn_validation
 from ..utils import STATIC_PATH, iscoroutinefunction, isgeneratorfunction
-from ..utils import get_free_port
+from ..utils import get_free_port, parse_file_size
 
 logger = logging.getLogger(__name__)
 
@@ -20,7 +21,7 @@ class FlaskHttpContext(HttpContext):
 
     def __init__(self):
         self.response = Response()
-        self.request_data = request.get_data()
+        self.request_data = request.data
 
     def request_obj(self):
         """返回当前请求对象"""
@@ -100,7 +101,9 @@ def start_server(applications, port=8080, host='', cdn=True, static_dir=None,
                  allowed_origins=None, check_origin=None,
                  session_expire_seconds=None,
                  session_cleanup_interval=None,
-                 debug=False, **flask_options):
+                 debug=False,
+                 max_payload_size='200M',
+                 **flask_options):
     """Start a Flask server to provide the PyWebIO application as a web service.
 
     :param int session_expire_seconds: Session expiration time, in seconds(default 600s).
@@ -109,6 +112,7 @@ def start_server(applications, port=8080, host='', cdn=True, static_dir=None,
        The server will periodically clean up expired sessions and release the resources occupied by the sessions.
     :param bool debug: Flask debug mode.
        If enabled, the server will automatically reload for code changes.
+    :param int/str max_payload_size: Max size of a request body which Flask can accept.
     :param flask_options: Additional keyword arguments passed to the ``flask.Flask.run``.
        For details, please refer: https://flask.palletsprojects.com/en/1.1.x/api/#flask.Flask.run
 
@@ -123,6 +127,7 @@ def start_server(applications, port=8080, host='', cdn=True, static_dir=None,
     cdn = cdn_validation(cdn, 'warn')
 
     app = Flask(__name__) if static_dir is None else Flask(__name__, static_url_path="/static", static_folder=static_dir)
+    utils.MAX_PAYLOAD_SIZE = app.config['MAX_CONTENT_LENGTH'] = parse_file_size(max_payload_size)
 
     app.add_url_rule('/', 'webio_view', webio_view(
         applications=applications, cdn=cdn,

pywebio/platform/path_deploy.py

@@ -1,17 +1,18 @@
 import os.path
+from functools import partial
 
 import tornado
 from tornado import template
 from tornado.web import HTTPError, Finish
 from tornado.web import StaticFileHandler
 
+from . import utils
 from .httpbased import HttpHandler
 from .tornado import webio_handler, set_ioloop
 from .tornado_http import TornadoHttpContext
 from .utils import cdn_validation, make_applications
 from ..session import register_session_implement, CoroutineBasedSession, ThreadBasedSession
 from ..utils import get_free_port, STATIC_PATH, parse_file_size
-from functools import partial
 
 
 def filename_ok(f):
@@ -132,17 +133,15 @@ def get_app_from_path(request_path, base, index, reload=False):
     return 'error', 404
 
 
-def _path_deploy(base, port=0, host='',
-                 static_dir=None, cdn=True, **tornado_app_settings):
+def _path_deploy(base, port=0, host='', static_dir=None, cdn=True, max_payload_size=2 ** 20 * 200,
+                 **tornado_app_settings):
     if not host:
         host = '0.0.0.0'
 
     if port == 0:
         port = get_free_port()
 
-    for k in list(tornado_app_settings.keys()):
-        if tornado_app_settings[k] is None:
-            del tornado_app_settings[k]
+    tornado_app_settings = {k: v for k, v in tornado_app_settings.items() if v is not None}
 
     abs_base = os.path.normpath(os.path.abspath(base))
 
@@ -165,7 +164,7 @@ def _path_deploy(base, port=0, host='',
 
     set_ioloop(tornado.ioloop.IOLoop.current())  # to enable bokeh app
     app = tornado.web.Application(handlers=handlers, **tornado_app_settings)
-    app.listen(port, address=host)
+    app.listen(port, address=host, max_buffer_size=max_payload_size)
     tornado.ioloop.IOLoop.current().start()
 
 
@@ -174,9 +173,7 @@ def path_deploy(base, port=0, host='',
                 reconnect_timeout=0,
                 cdn=True, debug=True,
                 allowed_origins=None, check_origin=None,
-                websocket_max_message_size=None,
-                websocket_ping_interval=None,
-                websocket_ping_timeout=None,
+                max_payload_size='200M',
                 **tornado_app_settings):
     """Deploy the PyWebIO applications from a directory.
 
@@ -198,12 +195,14 @@ def path_deploy(base, port=0, host='',
 
     The rest arguments of ``path_deploy()`` have the same meaning as for :func:`pywebio.platform.tornado.start_server`
     """
+
+    utils.MAX_PAYLOAD_SIZE = max_payload_size = parse_file_size(max_payload_size)
+    tornado_app_settings.setdefault('websocket_max_message_size', max_payload_size)  # Backward compatible
+    tornado_app_settings['websocket_max_message_size'] = parse_file_size(tornado_app_settings['websocket_max_message_size'])
     gen = _path_deploy(base, port=port, host=host,
                        static_dir=static_dir,
                        cdn=cdn, debug=debug,
-                       websocket_max_message_size=websocket_max_message_size,
-                       websocket_ping_interval=websocket_ping_interval,
-                       websocket_ping_timeout=parse_file_size(websocket_ping_timeout or '10M'),
+                       max_payload_size=max_payload_size,
                        **tornado_app_settings)
 
     cdn_url, abs_base = next(gen)
@@ -238,6 +237,7 @@ def path_deploy_http(base, port=0, host='',
                      allowed_origins=None, check_origin=None,
                      session_expire_seconds=None,
                      session_cleanup_interval=None,
+                     max_payload_size='200M',
                      **tornado_app_settings):
     """Deploy the PyWebIO applications from a directory.
 
@@ -248,9 +248,12 @@ def path_deploy_http(base, port=0, host='',
 
     The rest arguments of ``path_deploy_http()`` have the same meaning as for :func:`pywebio.platform.tornado_http.start_server`
     """
+    utils.MAX_PAYLOAD_SIZE = max_payload_size = parse_file_size(max_payload_size)
+
     gen = _path_deploy(base, port=port, host=host,
                        static_dir=static_dir,
                        cdn=cdn, debug=debug,
+                       max_payload_size=max_payload_size,
                        **tornado_app_settings)
 
     cdn_url, abs_base = next(gen)

pywebio/platform/tornado.py

@@ -16,6 +16,7 @@ import tornado.ioloop
 from tornado.web import StaticFileHandler
 from tornado.websocket import WebSocketHandler
 
+from . import utils
 from .utils import make_applications, render_page, cdn_validation, deserialize_binary_event
 from ..session import CoroutineBasedSession, ThreadBasedSession, ScriptModeSession, \
     register_session_implement_for_target, Session
@@ -255,7 +256,8 @@ async def open_webbrowser_on_server_started(host, port):
         logger.error('Open %s failed.' % url)
 
 
-def _setup_server(webio_handler, port=0, host='', static_dir=None, **tornado_app_settings):
+def _setup_server(webio_handler, port=0, host='', static_dir=None, max_buffer_size=2 ** 20 * 200,
+                  **tornado_app_settings):
     if port == 0:
         port = get_free_port()
 
@@ -267,7 +269,8 @@ def _setup_server(webio_handler, port=0, host='', static_dir=None, **tornado_app
     handlers.append((r"/(.*)", StaticFileHandler, {"path": STATIC_PATH, 'default_filename': 'index.html'}))
 
     app = tornado.web.Application(handlers=handlers, **tornado_app_settings)
-    server = app.listen(port, address=host)
+    # Credit: https://stackoverflow.com/questions/19074972/content-length-too-long-when-uploading-file-using-tornado
+    server = app.listen(port, address=host, max_buffer_size=max_buffer_size)
     return server, port
 
 
@@ -276,9 +279,7 @@ def start_server(applications, port=0, host='',
                  reconnect_timeout=0,
                  allowed_origins=None, check_origin=None,
                  auto_open_webbrowser=False,
-                 websocket_max_message_size=None,
-                 websocket_ping_interval=None,
-                 websocket_ping_timeout=None,
+                 max_payload_size='200M',
                  **tornado_app_settings):
     """Start a Tornado server to provide the PyWebIO application as a web service.
 
@@ -328,36 +329,27 @@ def start_server(applications, port=0, host='',
        It receives the source string (which contains protocol, host, and port parts) as parameter and return ``True/False`` to indicate that the server accepts/rejects the request.
        If ``check_origin`` is set, the ``allowed_origins`` parameter will be ignored.
     :param bool auto_open_webbrowser: Whether or not auto open web browser when server is started (if the operating system allows it) .
-    :param int/str websocket_max_message_size: Max bytes of a message which Tornado can accept.
-        Messages larger than the ``websocket_max_message_size`` (default 10MB) will not be accepted.
-        ``websocket_max_message_size`` can be a integer indicating the number of bytes, or a string ending with `K` / `M` / `G`
+    :param int/str max_payload_size: Max size of a websocket message which Tornado can accept.
+        Messages larger than the ``max_payload_size`` (default 200MB) will not be accepted.
+        ``max_payload_size`` can be a integer indicating the number of bytes, or a string ending with `K` / `M` / `G`
         (representing kilobytes, megabytes, and gigabytes, respectively).
         E.g: ``500``, ``'40K'``, ``'3M'``
-    :param int websocket_ping_interval: If set to a number, all websockets will be pinged every n seconds.
-        This can help keep the connection alive through certain proxy servers which close idle connections,
-        and it can detect if the websocket has failed without being properly closed.
-    :param int websocket_ping_timeout: If the ping interval is set, and the server doesn’t receive a ‘pong’
-        in this many seconds, it will close the websocket. The default is three times the ping interval,
-        with a minimum of 30 seconds. Ignored if ``websocket_ping_interval`` is not set.
     :param tornado_app_settings: Additional keyword arguments passed to the constructor of ``tornado.web.Application``.
         For details, please refer: https://www.tornadoweb.org/en/stable/web.html#tornado.web.Application.settings
     """
-    if websocket_max_message_size:
-        websocket_max_message_size = parse_file_size(websocket_max_message_size)
-    kwargs = locals()
-
     set_ioloop(tornado.ioloop.IOLoop.current())  # to enable bokeh app
 
-    app_options = ['debug', 'websocket_max_message_size', 'websocket_ping_interval', 'websocket_ping_timeout']
-    for opt in app_options:
-        if kwargs[opt] is not None:
-            tornado_app_settings[opt] = kwargs[opt]
-
     cdn = cdn_validation(cdn, 'warn')  # if CDN is not available, warn user and disable CDN
 
+    utils.MAX_PAYLOAD_SIZE = max_payload_size = parse_file_size(max_payload_size)
+
+    tornado_app_settings.setdefault('websocket_max_message_size', max_payload_size)  # Backward compatible
+    tornado_app_settings['websocket_max_message_size'] = parse_file_size(tornado_app_settings['websocket_max_message_size'])
+    tornado_app_settings['debug'] = debug
     handler = webio_handler(applications, cdn, allowed_origins=allowed_origins, check_origin=check_origin,
                             reconnect_timeout=reconnect_timeout)
-    _, port = _setup_server(webio_handler=handler, port=port, host=host, static_dir=static_dir, **tornado_app_settings)
+    _, port = _setup_server(webio_handler=handler, port=port, host=host, static_dir=static_dir,
+                            max_buffer_size=max_payload_size, **tornado_app_settings)
 
     print('Listen on %s:%s' % (host or '0.0.0.0', port))
 
@@ -454,7 +446,8 @@ def start_server_in_current_thread_session():
         if os.environ.get("PYWEBIO_SCRIPT_MODE_PORT"):
             port = int(os.environ.get("PYWEBIO_SCRIPT_MODE_PORT"))
 
-        server, port = _setup_server(webio_handler=SingleSessionWSHandler, port=port, host='localhost')
+        server, port = _setup_server(webio_handler=SingleSessionWSHandler, port=port, host='localhost',
+                                     websocket_max_message_size=parse_file_size('4G'))
         tornado.ioloop.IOLoop.current().spawn_callback(partial(wait_to_stop_loop, server=server))
 
         if "PYWEBIO_SCRIPT_MODE_PORT" not in os.environ:

pywebio/platform/tornado_http.py

@@ -4,10 +4,11 @@ import logging
 import tornado.ioloop
 import tornado.web
 
+from . import utils
 from .httpbased import HttpContext, HttpHandler
 from .tornado import set_ioloop, _setup_server, open_webbrowser_on_server_started
 from .utils import cdn_validation
-from ..utils import get_free_port
+from ..utils import parse_file_size
 
 logger = logging.getLogger(__name__)
 
@@ -111,9 +112,7 @@ def start_server(applications, port=8080, host='',
                  auto_open_webbrowser=False,
                  session_expire_seconds=None,
                  session_cleanup_interval=None,
-                 websocket_max_message_size=None,
-                 websocket_ping_interval=None,
-                 websocket_ping_timeout=None,
+                 max_payload_size='200M',
                  **tornado_app_settings):
     """Start a Tornado server to provide the PyWebIO application as a web service.
 
@@ -123,37 +122,36 @@ def start_server(applications, port=8080, host='',
        If no client message is received within ``session_expire_seconds``, the session will be considered expired.
     :param int session_cleanup_interval: Session cleanup interval, in seconds(default 120s).
        The server will periodically clean up expired sessions and release the resources occupied by the sessions.
+    :param int/str max_payload_size: Max size of a request body which Tornado can accept.
 
     The rest arguments of ``start_server()`` have the same meaning as for :func:`pywebio.platform.tornado.start_server`
 
     .. versionadded:: 1.2
     """
 
-    if port == 0:
-        port = get_free_port()
-
     if not host:
         host = '0.0.0.0'
 
     cdn = cdn_validation(cdn, 'warn')
 
-    kwargs = locals()
-
     set_ioloop(tornado.ioloop.IOLoop.current())  # to enable bokeh app
 
-    app_options = ['debug', 'websocket_max_message_size', 'websocket_ping_interval', 'websocket_ping_timeout']
-    for opt in app_options:
-        if kwargs[opt] is not None:
-            tornado_app_settings[opt] = kwargs[opt]
-
     cdn = cdn_validation(cdn, 'warn')  # if CDN is not available, warn user and disable CDN
 
+    utils.MAX_PAYLOAD_SIZE = max_payload_size = parse_file_size(max_payload_size)
+
+    tornado_app_settings.setdefault('websocket_max_message_size', max_payload_size)
+    tornado_app_settings['websocket_max_message_size'] = parse_file_size(
+        tornado_app_settings['websocket_max_message_size'])
+    tornado_app_settings['debug'] = debug
+
     handler = webio_handler(applications, cdn,
                             session_expire_seconds=session_expire_seconds,
                             session_cleanup_interval=session_cleanup_interval,
                             allowed_origins=allowed_origins, check_origin=check_origin)
 
-    _, port = _setup_server(webio_handler=handler, port=port, host=host, static_dir=static_dir, **tornado_app_settings)
+    _, port = _setup_server(webio_handler=handler, port=port, host=host, static_dir=static_dir,
+                            max_buffer_size=parse_file_size(max_payload_size), **tornado_app_settings)
 
     print('Listen on %s:%s' % (host or '0.0.0.0', port))
     if auto_open_webbrowser:

pywebio/platform/utils.py

@@ -14,6 +14,13 @@ from ..exceptions import PyWebIOWarning
 from ..utils import isgeneratorfunction, iscoroutinefunction, get_function_name, get_function_doc, \
     get_function_seo_info
 
+"""
+The maximum size in bytes of a http request body or a websocket message, after which the request or websocket is aborted
+Set by `start_server()` or `path_deploy()` 
+Used in `file_upload()` as the `max_size`/`max_total_size` parameter default or to validate the parameter. 
+"""
+MAX_PAYLOAD_SIZE = 0
+
 DEFAULT_CDN = "https://cdn.jsdelivr.net/gh/wang0618/PyWebIO-assets@v{version}/"
 
 AppMeta = namedtuple('App', 'title description')