diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ede2e0..be7cfb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python-version: [ '3.6', '3.7', '3.8', '3.9' ] + python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10' ] steps: - name: Checkout repo @@ -23,7 +23,7 @@ jobs: python -m pip install -r requirements.txt python setup.py install - name: Check source files - if: matrix.python-version == '3.9' + if: matrix.python-version == '3.10' run: | python -m pip install pytest-pycodestyle python -m pip install pytest-flakes diff --git a/ftw/http.py b/ftw/http.py index 58e3366..68da291 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -9,8 +9,8 @@ import socket import ssl import sys -import time import zlib +import select import brotli from IPy import IP @@ -19,8 +19,7 @@ from . import util -# Fallback to PROTOCOL_SSLv23 if PROTOCOL_TLS is not available. -PROTOCOL_TLS = getattr(ssl, 'PROTOCOL_TLS', ssl.PROTOCOL_SSLv23) +SOCKET_TIMEOUT = .3 class HttpResponse(object): @@ -267,7 +266,6 @@ def __init__(self): 'ADH-AES256-SHA:ECDHE-ECDSA-AES128-GCM-SHA256:' \ 'ECDHE-RSA-AES128-GCM-SHA256:AES128-GCM-SHA256:AES128-SHA256:HIGH:' self.CRLF = '\r\n' - self.HTTP_TIMEOUT = .3 self.RECEIVE_BYTES = 8192 self.SOCKET_TIMEOUT = 5 @@ -299,8 +297,9 @@ def build_socket(self): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Check if TLS if self.request_object.protocol == 'https': - context = ssl.SSLContext(PROTOCOL_TLS) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.set_ciphers(self.CIPHERS) + context.load_default_certs(ssl.Purpose.SERVER_AUTH) self.sock = context.wrap_socket( self.sock, server_hostname=self.request_object.dest_addr) self.sock.connect( @@ -466,38 +465,77 @@ def get_response(self): """ Get the response from the socket """ - self.sock.setblocking(0) our_data = [] - # Beginning time - begin = time.time() + self.sock.setblocking(False) + try: + our_data = self.read_response_from_socket() + finally: + try: + self.sock.shutdown(socket.SHUT_WR) + self.sock.close() + except OSError as err: + raise errors.TestError( + 'We were unable to close the socket as expected.', + { + 'msg': err, + 'function': 'http.HttpUA.get_response' + }) + else: + self.response_object = HttpResponse(b''.join(our_data), self) + finally: + if not b''.join(our_data): + raise errors.TestError( + 'No response from server.' + + ' Request likely timed out.', + { + 'host': self.request_object.dest_addr, + 'port': self.request_object.port, + 'proto': self.request_object.protocol, + 'msg': 'Please send the request and check' + + ' Wireshark', + 'function': 'http.HttpUA.get_response' + }) + + def read_response_from_socket(self): + # wait for socket to become ready + ready_sock, _, _ = select.select( + [self.sock], [], [self.sock], self.SOCKET_TIMEOUT) + if not ready_sock: + raise errors.TestError( + f'No response from server within {self.SOCKET_TIMEOUT}s', + { + 'host': self.request_object.dest_addr, + 'port': self.request_object.port, + 'proto': self.request_object.protocol, + 'msg': 'Please send the request and check Wireshark', + 'function': 'http.HttpUA.get_response' + }) + + our_data = [] while True: - # If we have data then if we're passed the timeout break - if our_data and time.time() - begin > self.HTTP_TIMEOUT: - break - # If we're dataless wait just a bit - elif time.time() - begin > self.HTTP_TIMEOUT * 2: - break - # Recv data try: data = self.sock.recv(self.RECEIVE_BYTES) - if data: - our_data.append(util.ensure_binary(data)) - begin = time.time() - else: - # Sleep for sometime to indicate a gap - time.sleep(self.HTTP_TIMEOUT) - except socket.error as err: - # Check if we got a timeout - if err.errno == errno.EAGAIN: - pass + if len(data) == 0: + # we're done + break + our_data.append(util.ensure_binary(data)) + except BlockingIOError as e: + # If we can't handle the error here, pass it on + if e.errno == socket.EAGAIN or e.errno == socket.EWOULDBLOCK: + # we're done + break + except OSError as err: # SSL will return SSLWantRead instead of EAGAIN - elif sys.platform == 'win32' and \ - err.errno == errno.WSAEWOULDBLOCK: + if (sys.platform == 'win32' and + err.errno == errno.WSAEWOULDBLOCK): pass elif (self.request_object.protocol == 'https' and - err.args[0] == ssl.SSL_ERROR_WANT_READ): - continue - # If we didn't it's an error + err.args[0] == ssl.SSL_ERROR_WANT_READ): + ready_sock, _, _ = select.select( + [self.sock], [], [self.sock], SOCKET_TIMEOUT) + if not ready_sock: + break + # It's an error else: raise errors.TestError( 'Failed to connect to server', @@ -508,26 +546,4 @@ def get_response(self): 'message': err, 'function': 'http.HttpUA.get_response' }) - try: - self.sock.shutdown(socket.SHUT_WR) - self.sock.close() - except socket.error as err: - raise errors.TestError( - 'We were unable to close the socket as expected.', - { - 'msg': err, - 'function': 'http.HttpUA.get_response' - }) - else: - self.response_object = HttpResponse(b''.join(our_data), self) - finally: - if not b''.join(our_data): - raise errors.TestError( - 'No response from server. Request likely timed out.', - { - 'host': self.request_object.dest_addr, - 'port': self.request_object.port, - 'proto': self.request_object.protocol, - 'msg': 'Please send the request and check Wireshark', - 'function': 'http.HttpUA.get_response' - }) + return our_data diff --git a/ftw/logchecker.py b/ftw/logchecker.py index 3f3d557..22e0fa5 100644 --- a/ftw/logchecker.py +++ b/ftw/logchecker.py @@ -15,6 +15,20 @@ def set_times(self, start, end): self.start = start self.end = end + def mark_start(self, stage_id): + """ + May be implemented to set up the log checker before + the request is being sent + """ + pass + + def mark_end(self, stage_id): + """ + May be implemented to tell the log checker that + the response has been received + """ + pass + @abstractmethod def get_logs(self): """ diff --git a/ftw/pytest_plugin.py b/ftw/pytest_plugin.py index d6ebf16..63d288c 100644 --- a/ftw/pytest_plugin.py +++ b/ftw/pytest_plugin.py @@ -7,7 +7,7 @@ from .ruleset import Test -def get_testdata(rulesets): +def get_testdata(rulesets, use_rulesets): """ In order to do test-level parametrization (is this a word?), we have to bundle the test data from rulesets into tuples so py.test can understand @@ -17,7 +17,10 @@ def get_testdata(rulesets): for ruleset in rulesets: for test in ruleset.tests: if test.enabled: - testdata.append((ruleset, test)) + args = [test] + if use_rulesets: + args = [rulesets] + args + testdata.append(args) return testdata @@ -127,7 +130,13 @@ def pytest_generate_tests(metafunc): metafunc.config.option.ruledir_recurse, True) if metafunc.config.option.rule: rulesets = util.get_rulesets(metafunc.config.option.rule, False) - if 'ruleset' in metafunc.fixturenames and \ - 'test' in metafunc.fixturenames: - metafunc.parametrize('ruleset, test', get_testdata(rulesets), - ids=test_id) + if 'test' in metafunc.fixturenames: + use_rulesets = False + arg_names = ['test'] + if 'ruleset' in metafunc.fixturenames: + use_rulesets = True + arg_names = ['ruleset'] + arg_names + metafunc.parametrize( + arg_names, + get_testdata(rulesets, use_rulesets), + ids=test_id) diff --git a/ftw/ruleset.py b/ftw/ruleset.py index 04f951c..13f06af 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -137,18 +137,26 @@ class Stage(object): This class holds information about 1 stage in a test, which contains 1 input and 1 output """ - def __init__(self, stage_dict): + def __init__(self, stage_dict, stage_index, test): self.stage_dict = stage_dict + self.stage_index = stage_index + self.test = test self.input = Input(**stage_dict['input']) self.output = Output(stage_dict['output']) + self.id = self.build_id() + + def build_id(self): + rule_name = self.test.ruleset_meta["name"].split('.')[0] + return f'{rule_name}-{self.test.test_index}-{self.stage_index}' class Test(object): """ This class holds information for 1 test and potentially many stages """ - def __init__(self, test_dict, ruleset_meta): + def __init__(self, test_dict, test_index, ruleset_meta): self.test_dict = test_dict + self.test_index = test_index self.ruleset_meta = ruleset_meta self.test_title = self.test_dict['test_title'] self.stages = self.build_stages() @@ -160,8 +168,8 @@ def build_stages(self): """ Processes and loads an array of stages from the test dictionary """ - return [Stage(stage_dict['stage']) - for stage_dict in self.test_dict['stages']] + return [Stage(stage_dict['stage'], index, self) + for index, stage_dict in enumerate(self.test_dict['stages'])] class Ruleset(object): @@ -183,8 +191,8 @@ def extract_tests(self): creates test objects based on input """ try: - return [Test(test_dict, self.meta) - for test_dict in self.yaml_file['tests']] + return [Test(test_dict, index, self.meta) + for index, test_dict in enumerate(self.yaml_file['tests'])] except errors.TestError as e: e.args[1]['meta'] = self.meta raise e diff --git a/ftw/testrunner.py b/ftw/testrunner.py index 5f1c907..21f6ade 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -174,7 +174,6 @@ def run_stage(self, stage, logger_obj=None, http_ua=None): input, waits for output then compares expected vs actual output http_ua can be passed in to persist cookies """ - # Send our request (exceptions caught as needed) if stage.output.expect_error: with pytest.raises(errors.TestError) as excinfo: @@ -187,11 +186,20 @@ def run_stage(self, stage, logger_obj=None, http_ua=None): else: if not http_ua: http_ua = http.HttpUA() - start = datetime.datetime.utcnow() + if ((stage.output.log_contains_str or + stage.output.no_log_contains_str) and + logger_obj is not None): + logger_obj.mark_start(stage.id) + start = datetime.datetime.utcnow() http_ua.send_request(stage.input) - end = datetime.datetime.utcnow() - if (stage.output.log_contains_str or - stage.output.no_log_contains_str) and logger_obj is not None: + if ((stage.output.log_contains_str or + stage.output.no_log_contains_str) and + logger_obj is not None): + logger_obj.mark_end(stage.id) + end = datetime.datetime.utcnow() + if ((stage.output.log_contains_str or + stage.output.no_log_contains_str) and + logger_obj is not None): logger_obj.set_times(start, end) lines = logger_obj.get_logs() if stage.output.log_contains_str: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..374b58c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index a5d2a37..b46f7dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -Brotli==1.0.7 -IPy==0.83 -PyYAML==4.2b1 -pytest==4.6 -python-dateutil==2.6.0 +Brotli==1.0.9 +IPy==1.01 +PyYAML==6.0 +pytest==6.2.5 +python-dateutil==2.8.2 diff --git a/setup.cfg b/setup.cfg index cc752ab..8b88600 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -description-file = README.md +description_file = README.md [tool:pytest] addopts = -s -v diff --git a/setup.py b/setup.py index 006a82b..59309d0 100644 --- a/setup.py +++ b/setup.py @@ -1,29 +1,44 @@ -#!/usr/bin/env python +import setuptools -from setuptools import setup +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() -setup(name='ftw', - description='Framework for Testing WAFs', - author='Chaim Sanders, Zack Allen', - author_email='zma4580@gmail.com, chaim.sanders@gmail.com', - url='https://www.github.com/coreruleset/ftw', - include_package_data=True, - package_data={ +setuptools.setup( + name='ftw', + description='Framework for Testing WAFs', + long_description=long_description, + long_description_content_type="text/markdown", + author='Chaim Sanders, Zack Allen', + author_email='zma4580@gmail.com, chaim.sanders@gmail.com', + url='https://github.com/coreruleset/ftw', + include_package_data=True, + package_data={ 'ftw': ['util/public_suffix_list.dat'] - }, - entry_points={ - 'pytest11': [ - 'ftw = ftw.pytest_plugin' - ] - }, - packages=['ftw'], - keywords=['waf'], - use_scm_version=True, - setup_requires=['setuptools_scm'], - install_requires=[ - 'Brotli==1.0.7', - 'IPy==0.83', - 'PyYAML==4.2b1', - 'pytest==4.6', - 'python-dateutil==2.6.0' - ]) + }, + entry_points={ + 'pytest11': [ + 'ftw = ftw.pytest_plugin' + ] + }, + keywords=['waf'], + project_urls={ + "Bug Tracker": 'https://github.com/coreruleset/ftw/issues', + }, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Framework :: Pytest", + ], + packages=["ftw"], + python_requires=">=3.6", + use_scm_version=True, + setup_requires=['setuptools_scm'], + install_requires=[ + 'Brotli==1.0.9', + 'IPy==1.01', + 'PyYAML==6.0', + 'pytest==6.2.5', + 'python-dateutil==2.8.2' + ], +) diff --git a/test/integration/COOKIEFIXTURE.yaml b/test/integration/COOKIEFIXTURE.yaml index e5676e6..f700204 100644 --- a/test/integration/COOKIEFIXTURE.yaml +++ b/test/integration/COOKIEFIXTURE.yaml @@ -1,40 +1,67 @@ --- - meta: - author: "Chaim" - enabled: true - name: "COOKIEFIXTURE.yaml" - description: "Tests cookie saving functionality" - tests: - - - test_title: "Multi-Stage w\\ Cookie" - stages: - - - stage: - input: - save_cookie: true - dest_addr: "www.ieee.org" - method: "GET" - port: 443 - headers: - User-Agent: "Foo" - Host: "www.ieee.org" - protocol: "https" - uri: "/" - output: - status: 200 - response_contains: "Set-Cookie: TS01247332=" - - - stage: - input: - save_cookie: true - dest_addr: "www.ieee.org" - method: "GET" - port: 443 - headers: - User-Agent: "Foo" - Host: "www.ieee.org" - protocol: "https" - uri: "/" - output: - status: 200 - response_contains: "Set-Cookie: TS01247332=" +meta: + author: "Chaim" + enabled: true + name: "COOKIEFIXTURE.yaml" + description: "Tests cookie saving functionality" +tests: + - test_title: "Multi-Stage w\\ Cookie" + stages: + - stage: + input: + save_cookie: true + dest_addr: "www.cloudflare.com" + method: "GET" + port: 443 + headers: + User-Agent: "Foo" + Host: "www.cloudflare.com" + protocol: "https" + uri: "/" + output: + status: 200 + response_contains: "[Ss]et-[Cc]ookie: __cf_bm=" + - stage: + input: + save_cookie: true + dest_addr: "www.cloudflare.com" + method: "GET" + port: 443 + headers: + User-Agent: "Foo" + Host: "www.cloudflare.com" + protocol: "https" + uri: "/" + output: + status: 200 + no_response_contains: "[Ss]et-[Cc]ookie: __cf_bm=" + - test_title: "Multi-Stage w\\ Cookie; failure because the cookie is reset if not all cookies are present and ftw can only handle one cookie header" + stages: + - stage: + input: + save_cookie: true + dest_addr: "www.ieee.org" + method: "GET" + port: 443 + headers: + User-Agent: "Foo" + Host: "www.ieee.org" + protocol: "https" + uri: "/" + output: + status: 200 + response_contains: "[Ss]et-[Cc]ookie: TS01247332=" + - stage: + input: + save_cookie: true + dest_addr: "www.ieee.org" + method: "GET" + port: 443 + headers: + User-Agent: "Foo" + Host: "www.ieee.org" + protocol: "https" + uri: "/" + output: + status: 200 + no_response_contains: "[Ss]et-[Cc]ookie: TS01247332=" diff --git a/test/integration/test_cookie.py b/test/integration/test_cookie.py index 0dd9f8a..bf36434 100644 --- a/test/integration/test_cookie.py +++ b/test/integration/test_cookie.py @@ -2,6 +2,11 @@ import pytest +@pytest.mark.skip( + reason=""" + 1. ieee.org has a very bad web server, so responses fail a lot + 2. ieee.org sends multiple set-cookie headers and ftw can only handle a single header of the same name""" +) def test_default(ruleset, test, destaddr): """ Default tester with no logger obj. Useful for HTML contains and Status code diff --git a/test/integration/test_http.py b/test/integration/test_http.py index d2bd8d1..dc656aa 100644 --- a/test/integration/test_http.py +++ b/test/integration/test_http.py @@ -3,21 +3,20 @@ import pytest -@pytest.mark.skip(reason='Integration failure, @chaimsanders for more info') def test_cookies1(): """Tests accessing a site that sets a cookie and then wants to resend the cookie""" http_ua = http.HttpUA() - x = ruleset.Input(protocol='https', port=443, dest_addr='www.ieee.org', - headers={'Host': 'www.ieee.org'}) + x = ruleset.Input(protocol='https', port=443, dest_addr='www.cloudflare.com', + headers={'Host': 'www.cloudflare.com'}) http_ua.send_request(x) with pytest.raises(KeyError): print(http_ua.request_object.headers['cookie']) assert('set-cookie' in list(http_ua.response_object.headers.keys())) cookie_data = http_ua.response_object.headers['set-cookie'] cookie_var = cookie_data.split('=')[0] - x = ruleset.Input(protocol='https', port=443, dest_addr='www.ieee.org', - headers={'Host': 'www.ieee.org'}) + x = ruleset.Input(protocol='https', port=443, dest_addr='www.cloudflare.com', + headers={'Host': 'www.cloudflare.com'}) http_ua.send_request(x) assert(http_ua.request_object.headers['cookie'].split('=')[0] == cookie_var) @@ -26,11 +25,11 @@ def test_cookies1(): def test_cookies2(): """Test to make sure that we don't override user specified cookies""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr='ieee.org', + x = ruleset.Input(dest_addr='example.com', headers={ - 'Host': 'ieee.org', + 'Host': 'example.com', 'cookie': 'TS01247332=012f3506234413e6c5cb14e8c0' 'd5bf890fdd02481614b01cd6cd30911c6733e' '3e6f79e72aa'}) @@ -44,11 +43,11 @@ def test_cookies3(): """Test to make sure we retain cookies when user specified values are provided""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr='ieee.org', + x = ruleset.Input(dest_addr='example.com', headers={ - 'Host': 'ieee.org', + 'Host': 'example.com', 'cookie': 'TS01247332=012f3506234413e6c5cb14e8c0d' '5bf890fdd02481614b01cd6cd30911c6733e3e' '6f79e72aa; XYZ=123'}) @@ -62,9 +61,9 @@ def test_cookies4(): """Test to make sure cookies are saved when user-specified cookie is added""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org', + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com', 'cookie': 'XYZ=123'}) http_ua.send_request(x) assert('XYZ' in http_ua.request_object.headers['cookie']) @@ -80,7 +79,6 @@ def test_raw1(): assert http_ua.response_object.status == 200 -@pytest.mark.skip(reason='Integration failure, @chaimsanders for more info') def test_raw2(): """Test to make sure a raw request will work with actual seperators""" x = ruleset.Input(dest_addr='example.com', raw_request='''GET / HTTP/1.1 diff --git a/test/integration/test_logcontains.py b/test/integration/test_logcontains.py index b76f77b..4549396 100644 --- a/test/integration/test_logcontains.py +++ b/test/integration/test_logcontains.py @@ -27,13 +27,13 @@ def logchecker_obj(): return LoggerTestObj() -def test_logcontains_withlog(logchecker_obj, ruleset, test): +def test_logcontains_withlog(logchecker_obj, test): runner = testrunner.TestRunner() for stage in test.stages: runner.run_stage(stage, logchecker_obj) -def test_logcontains_nolog(logchecker_obj, ruleset, test): +def test_logcontains_nolog(logchecker_obj, test): logchecker_obj.do_nothing = True runner = testrunner.TestRunner() with(pytest.raises(AssertionError)): diff --git a/test/unit/test_ruleset.py b/test/unit/test_ruleset.py index ac15c97..763773a 100644 --- a/test/unit/test_ruleset.py +++ b/test/unit/test_ruleset.py @@ -33,11 +33,12 @@ def test_input(): def test_testobj(): with pytest.raises(KeyError) as excinfo: - ruleset.Test({}, {}) + ruleset.Test({}, {}, {}) assert 'test_title' in str(excinfo.value) + ruleset_meta = {'name': 'test-name.yaml'} stages_dict = {'test_title': 1, 'stages': [{'stage': {'output': {'log_contains': 'foo'}, 'input': {}}}]} - ruleset.Test(stages_dict, {}) + ruleset.Test(stages_dict, {}, ruleset_meta) def test_ruleset():