tableau · 0golovatyi · Oct 19, 2019 · Oct 4, 2019 · Oct 4, 2019 · Oct 4, 2019
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,5 +1,12 @@
 # Changelog
 
+## v0.8.9
+
+### Improvements
+
+- Added Ctrl+C handler
+- Added configurable buffer size for HTTP requests
+
 ## v0.8.7
 
 ### Improvements

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -177,3 +177,15 @@ TabPy package:
 python setup.py sdist bdist_wheel
 python -m twine upload dist/*
 ```
+
+To publish test version of the package use the following command:
+
+```sh
+python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+```
+
+To install package from TestPyPi use the command:
+
+```sh
+pip install -i https://test.pypi.org/simple/ tabpy
+```
diff --git a/docs/server-config.md b/docs/server-config.md
@@ -85,6 +85,9 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log
   not set.
 - `TABPY_LOG_DETAILS` - when set to `true` additional call information
   (caller IP, URL, client info, etc.) is logged. Default value - `false`.
+- `TABPY_MAX_REQUEST_SIZE_MB` - maximal request size supported by TabPy server
+  in Megabytes. All requests of exceeding size are rejected. Default value is
+  100 Mb.
 - `TABPY_EVALUATE_TIMEOUT` - script evaluation timeout in seconds. Default
   value - `30`.
 
@@ -116,10 +119,15 @@ settings._
 # end user info if provided.
 # TABPY_LOG_DETAILS = true
 
+# Limit request size (in Mb) - any request which size exceeds
+# specified amount will be rejected by TabPy.
+# Default value is 100 Mb.
+# TABPY_MAX_REQUEST_SIZE_MB = 100
+
 # Configure how long a custom script provided to the /evaluate method
 # will run before throwing a TimeoutError.
 # The value should be a float representing the timeout time in seconds.
-#TABPY_EVALUATE_TIMEOUT = 30
+# TABPY_EVALUATE_TIMEOUT = 30
 
 [loggers]
 keys=root

diff --git a/tabpy/VERSION b/tabpy/VERSION
@@ -1 +1 @@
-0.8.7
+0.8.9
diff --git a/tabpy/tabpy_server/app/ConfigParameters.py b/tabpy/tabpy_server/app/ConfigParameters.py
@@ -12,4 +12,5 @@ class ConfigParameters:
     TABPY_PWD_FILE = 'TABPY_PWD_FILE'
     TABPY_LOG_DETAILS = 'TABPY_LOG_DETAILS'
     TABPY_STATIC_PATH = 'TABPY_STATIC_PATH'
+    TABPY_MAX_REQUEST_SIZE_MB = 'TABPY_MAX_REQUEST_SIZE_MB'
     TABPY_EVALUATE_TIMEOUT = 'TABPY_EVALUATE_TIMEOUT'
diff --git a/tabpy/tabpy_server/app/SettingsParameters.py b/tabpy/tabpy_server/app/SettingsParameters.py
@@ -12,4 +12,5 @@ class SettingsParameters:
     ApiVersions = 'versions'
     LogRequestContext = 'log_request_context'
     StaticPath = 'static_path'
+    MaxRequestSizeInMb = 'max_request_size_in_mb'
     EvaluateTimeout = 'evaluate_timeout'
diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py
@@ -6,6 +6,7 @@
 import multiprocessing
 import os
 import shutil
+import signal
 import tabpy.tabpy_server
 from tabpy.tabpy import __version__
 from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters
@@ -60,32 +61,51 @@ def __init__(self, config_file=None):
 
     def run(self):
         application = self._create_tornado_web_app()
+        max_request_size =\
+            int(self.settings[SettingsParameters.MaxRequestSizeInMb]) * 1024 * 1024
+        logger.info(f'Setting max request size to {max_request_size} bytes')
 
         init_model_evaluator(
             self.settings,
             self.tabpy_state,
             self.python_service)
 
         protocol = self.settings[SettingsParameters.TransferProtocol]
-        if protocol == 'http':
-            application.listen(self.settings[SettingsParameters.Port])
-        elif protocol == 'https':
-            application.listen(self.settings[SettingsParameters.Port],
-                               ssl_options={
+        ssl_options = None
+        if protocol == 'https':
+            ssl_options = {
                 'certfile': self.settings[SettingsParameters.CertificateFile],
                 'keyfile': self.settings[SettingsParameters.KeyFile]
-            })
-        else:
+            }
+        elif protocol != 'http':
             msg = f'Unsupported transfer protocol {protocol}.'
             logger.critical(msg)
             raise RuntimeError(msg)
 
+        application.listen(
+            self.settings[SettingsParameters.Port],
+            ssl_options=ssl_options,
+            max_buffer_size=max_request_size,
+            max_body_size=max_request_size)
+
         logger.info(
             'Web service listening on port '
             f'{str(self.settings[SettingsParameters.Port])}')
         tornado.ioloop.IOLoop.instance().start()
 
     def _create_tornado_web_app(self):
+        class TabPyTornadoApp(tornado.web.Application):
+            is_closing = False
+
+            def signal_handler(self, signal, frame):
+                logger.critical(f'Exiting on signal {signal}...')
+                self.is_closing = True
+
+            def try_exit(self):
+                if self.is_closing:
+                    tornado.ioloop.IOLoop.instance().stop()
+                    logger.info('Shutting down TabPy...')
+
        logger.info('Initializing TabPy...')
         tornado.ioloop.IOLoop.instance().run_sync(
             lambda: init_ps_server(self.settings, self.tabpy_state))
@@ -95,7 +115,7 @@ def _create_tornado_web_app(self):
             max_workers=multiprocessing.cpu_count())
 
         # initialize Tornado application
-        application = tornado.web.Application([
+        application = TabPyTornadoApp([
             # skip MainHandler to use StaticFileHandler .* page requests and
             # default to index.html
             # (r"/", MainHandler),
@@ -121,6 +141,9 @@ def _create_tornado_web_app(self):
                   default_filename="index.html")),
         ], debug=False, **self.settings)
 
+        signal.signal(signal.SIGINT, application.signal_handler)
+        tornado.ioloop.PeriodicCallback(application.try_exit, 500).start()
+
         return application
 
     @staticmethod
@@ -303,6 +326,10 @@ def set_parameter(settings_key,
             else 'disabled'
         logger.info(f'Call context logging is {call_context_state}')
 
+        set_parameter(SettingsParameters.MaxRequestSizeInMb,
+                      ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB,
+                      default_val=100)
+
     def _validate_transfer_protocol_settings(self):
         if SettingsParameters.TransferProtocol not in self.settings:
             msg = 'Missing transfer protocol information.'

diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf
@@ -20,10 +20,15 @@
 # end user info if provided.
 # TABPY_LOG_DETAILS = true
 
+# Limit request size (in Mb) - any request which size exceeds
+# specified amount will be rejected by TabPy.
+# Default value is 100 Mb.
+# TABPY_MAX_REQUEST_SIZE_MB = 100
+
 # Configure how long a custom script provided to the /evaluate method
 # will run before throwing a TimeoutError.
 # The value should be a float representing the timeout time in seconds.
-#TABPY_EVALUATE_TIMEOUT = 30
+# TABPY_EVALUATE_TIMEOUT = 30
 
 [loggers]
 keys=root

diff --git a/tabpy/tabpy_server/handlers/management_handler.py b/tabpy/tabpy_server/handlers/management_handler.py
@@ -94,7 +94,7 @@ def _add_or_update_endpoint(self, action, name, version, request_data):
                 self.settings[SettingsParameters.StateFilePath], name, version)
             self.logger.log(logging.DEBUG,
                             f'Checking source path {src_path}...')
-            _path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/\.]+$')
+            _path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/\.\(\)]+$')
             # copy from staging
             if src_path:
                 if not isinstance(request_data['src_path'], str):

diff --git a/tabpy/tabpy_server/handlers/query_plane_handler.py b/tabpy/tabpy_server/handlers/query_plane_handler.py
@@ -135,6 +135,7 @@ def _process_query(self, endpoint_name, start):
             # Sanitize input data
             data = self._sanitize_request_data(json.loads(request_json))
         except Exception as e:
+            self.logger.log(logging.ERROR, str(e))
             err_msg = format_exception(e, "Invalid Input Data")
             self.error_out(400, err_msg)
             return
@@ -177,6 +178,7 @@ def _process_query(self, endpoint_name, start):
                 return
 
         except Exception as e:
+            self.logger.log(logging.ERROR, str(e))
             err_msg = format_exception(e, 'process query')
             self.error_out(500, 'Error processing query', info=err_msg)
             return

diff --git a/tabpy/tabpy_server/psws/python_service.py b/tabpy/tabpy_server/psws/python_service.py
@@ -42,6 +42,7 @@ def manage_request(self, msg):
             logger.debug(f'Returning response {response}')
             return response
         except Exception as e:
+            logger.exception(e)
             msg = e
             if hasattr(e, 'message'):
                 msg = e.message
@@ -90,6 +91,7 @@ def _load_object(self, object_uri, object_url, object_version, is_update,
                                               'status': 'LoadSuccessful',
                                               'last_error': None}
         except Exception as e:
+            logger.exception(e)
             logger.error(f'Unable to load QueryObject: path={object_url}, '
                          f'error={str(e)}')
 
@@ -132,6 +134,7 @@ def load_object(self, object_uri, object_url, object_version, is_update,
                     object_uri, object_url, object_version, is_update,
                     object_type)
         except Exception as e:
+            logger.exception(e)
             logger.error(f'Unable to load QueryObject: path={object_url}, '
                          f'error={str(e)}')
 
@@ -226,6 +229,7 @@ def query(self, object_uri, params, uid):
             else:
                 return UnknownURI(object_uri)
         except Exception as e:
+            logger.exception(e)
             err_msg = format_exception(e, '/query')
             logger.error(err_msg)
             return QueryFailed(uri=object_uri, error=err_msg)
diff --git a/tabpy/tabpy_tools/rest.py b/tabpy/tabpy_tools/rest.py
@@ -1,12 +1,11 @@
 import abc
+from collections.abc import MutableMapping
 import logging
 import requests
 from requests.auth import HTTPBasicAuth
 from re import compile
    import json as json
 
-from collections import MutableMapping as _MutableMapping
-
 
 logger = logging.getLogger(__name__)
 
@@ -290,7 +289,7 @@ def __init__(self, name, bases, dict):
                 self.__rest__.add(k)
 
 
-class RESTObject(_MutableMapping, metaclass=_RESTMetaclass):
+class RESTObject(MutableMapping, metaclass=_RESTMetaclass):
     """A base class that has methods generally useful for interacting with
     REST objects. The attributes are accessible either as dict keys or as
     attributes. The object also behaves like a dict, even replicating the

diff --git a/tabpy/tabpy_tools/schema.py b/tabpy/tabpy_tools/schema.py
@@ -1,5 +1,5 @@
 import logging
-import genson as _genson
+import genson
 import jsonschema
 
 
@@ -12,9 +12,9 @@ def _generate_schema_from_example_and_description(input, description):
     to the example in json-schema.org. The description given by the users
     is then added to the schema.
     '''
-    s = _genson.Schema()
+    s = genson.SchemaBuilder(None)
     s.add_object(input)
-    input_schema = s.to_dict()
+    input_schema = s.to_schema()
 
     if description is not None:
         if 'properties' in input_schema:

diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py
@@ -227,20 +227,19 @@ def setUp(self):
         with open(self.tmp_dir + '/output.txt', 'w') as outfile:
             cmd = ['tabpy',
                    '--config=' + self.config_file_name]
-            coverage.process_startup()
+            preexec_fn = None
             if platform.system() == 'Windows':
                 self.py = 'python'
-                self.process = subprocess.Popen(
-                    cmd,
-                    stdout=outfile,
-                    stderr=outfile)
             else:
                 self.py = 'python3'
-                self.process = subprocess.Popen(
-                    cmd,
-                    preexec_fn=os.setsid,
-                    stdout=outfile,
-                    stderr=outfile)
+                preexec_fn = os.setsid
+
+            coverage.process_startup()
+            self.process = subprocess.Popen(
+                cmd,
+                preexec_fn=preexec_fn,
+                stdout=outfile,
+                stderr=outfile)
 
             # give the app some time to start up...
             time.sleep(5)
@@ -299,3 +298,7 @@ def deploy_models(self, username: str, password: str):
                 input=input_string.encode('utf-8'),
                 stdout=outfile,
                 stderr=outfile)
+
+    def _get_process(self):
+        return self.process
+
diff --git a/tests/integration/test_app.py b/tests/integration/test_app.py
@@ -0,0 +1,16 @@
+import integ_test_base
+import os
+import signal
+import unittest
+
+class TestApp(integ_test_base.IntegTestBase):
+    def test_ctrl_c(self):
+        # Uncomment the following line to preserve
+        # test case output and other files (config, state, ect.)
+        # in system temp folder.
+        # self.set_delete_temp_folder(False)
+
+        process = self._get_process()
+        os.kill(process.pid, signal.SIGINT)
+
+
diff --git a/tests/integration/test_deploy_and_evaluate_model.py b/tests/integration/test_deploy_and_evaluate_model.py
@@ -14,7 +14,7 @@ def test_deploy_and_evaluate_model(self):
         # Uncomment the following line to preserve
         # test case output and other files (config, state, ect.)
         # in system temp folder.
-        self.set_delete_temp_folder(False)
+        # self.set_delete_temp_folder(False)
 
         self.deploy_models(self._get_username(), self._get_password())
 

diff --git a/tests/integration/test_deploy_and_evaluate_model_ssl.py b/tests/integration/test_deploy_and_evaluate_model_ssl.py
@@ -18,6 +18,11 @@ def _get_key_file_name(self) -> str:
         return './tests/integration/resources/2019_04_24_to_3018_08_25.key'
 
     def test_deploy_and_evaluate_model_ssl(self):
+        # Uncomment the following line to preserve
+        # test case output and other files (config, state, ect.)
+        # in system temp folder.
+        self.set_delete_temp_folder(False)
+
         self.deploy_models(self._get_username(), self._get_password())
 
         payload = (

diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py
@@ -10,6 +10,19 @@
 
 
 class TestConfigEnvironmentCalls(unittest.TestCase):
+    def test_config_file_does_not_exist(self):
+        app = TabPyApp('/folder_does_not_exit/file_does_not_exist.conf')
+
+        self.assertEqual(app.settings['port'], 9004)
+        self.assertEqual(app.settings['server_version'],
+                         open('tabpy/VERSION').read().strip())
+        self.assertEqual(app.settings['transfer_protocol'], 'http')
+        self.assertTrue('certificate_file' not in app.settings)
+        self.assertTrue('key_file' not in app.settings)
+        self.assertEqual(app.settings['log_request_context'], False)
+        self.assertEqual(app.settings['evaluate_timeout'], 30)
+
+
     @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments',
            return_value=Namespace(config=None))
     @patch('tabpy.tabpy_server.app.app.TabPyState')