coreruleset · fzipi · Nov 26, 2021 · Oct 29, 2021 · Oct 29, 2021 · Nov 1, 2021
diff --git a/tests/regression/README.md b/tests/regression/README.md
@@ -35,7 +35,8 @@ SecAction "id:900005,\
   setvar:tx.paranoia_level=4,\
   setvar:tx.crs_validate_utf8_encoding=1,\
   setvar:tx.arg_name_length=100,\
-  setvar:tx.arg_length=400"
+  setvar:tx.arg_length=400,\
+  setvar:tx.combined_file_sizes=65535"
 ```
 
 Once these requirements have been met the tests can be run by using pytest.

diff --git a/util/find-max-datalen-in-tests/README.md b/util/find-max-datalen-in-tests/README.md
@@ -0,0 +1,47 @@
+# Find the longest data in CRS test cases
+
+This page describes how can you find the longest data string in CRS test cases.
+
+## Goals
+
+Some rules check the `FILES_COMBINED_SIZE` against the `TX:COMBINED_FILE_SIZES` variable. To check these work as well, we need to set the `tx.combined_file_sizes` variable and send a payload which is greater than this value - see [this](https://github.com/coreruleset/coreruleset/blob/v3.4/dev/tests/regression/README.md#requirements):
+
+```
+SecAction "id:900005,\
+  phase:1,\
+  nolog,\
+  pass,\
+  ctl:ruleEngine=DetectionOnly,\
+  ctl:ruleRemoveById=910000,\
+  setvar:tx.paranoia_level=4,\
+  setvar:tx.crs_validate_utf8_encoding=1,\
+  setvar:tx.arg_name_length=100,\
+  setvar:tx.arg_length=400,\
+  setvar:tx.combined_file_sizes=MAX_LEN"
+```
+
+In `modsecurity-crs-docker` [here](https://github.com/coreruleset/modsecurity-crs-docker/blob/master/src/opt/modsecurity/activate-rules.sh#L79-L82) is how the setting works.
+
+To configure the Github action, you need to set up this in CORERULESET/test/docker-compose.yaml:
+
+```
+   ...
+   COMBINED_FILE_SIZES=MAX_LEN
+   ...
+```
+
+## Usage
+
+To find the possible value of MAX_LEN, run this script with one mandatory, and one optional argument. The mandatory argument is the path of the CRS directory (the root). The optional argument is the `-i` or `--ignoretests`, where you can pass the test id what you want to skip - eg. you want to use the test to exceed the maximum length.
+
+```
+./find_max_datalen.py ../.. -i 920410-1
+```
+
+In this case, you pass the CRS root as parent dir, and skip the test id 920410-1, which wants to exceed the maximum length.
+
+## Prerequisites
+
+* Python3 interpreter
+* Py-YAML
+* CRS rule set
diff --git a/util/find-max-datalen-in-tests/find_max_datalen.py b/util/find-max-datalen-in-tests/find_max_datalen.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# This file helps to find the longest data size in all test cases under
+# CORERULESET_ROOT/test/regression/tests directory.
+
+# You just have to pass the CORERULESET_ROOT as argument.
+# Optional argument can be passed -i or --ignoretests - the listed test
+# cases will skipped.
+
+# At the end, the script will print the longest length, and the rule where
+# the data is.
+
+
+import sys
+import os
+import os.path
+import yaml
+import argparse
+
+if __name__ == "__main__":
+
+    desc = """This script needs a mandatory argument where you pass the path to your
+coreruleset. Then it iterates through tests, and finds the longest request
+body (data) between test cases. To ignore a test case, pass the number of the
+test with '-i' or '--ignoretests', eg.: '... -i 920410-1'"""
+
+    parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('-i', '--ignoretests', metavar='ignoretests',
+                        help='Ignore listed rules, separated by comma', required=False,
+                        nargs=1)
+    parser.add_argument('crspath', metavar='/path/to/coreruleset', type=str,
+                        help
8000
='Directory path to CRS')
+    args = parser.parse_args()
+
+    test_cases = {}
+    testpath = args.crspath.rstrip("/") + "/tests/regression/tests"
+
+    if not os.path.isdir(testpath):
+        print("Directory does not exist: %s" % (testpath))
+        sys.exit(1)
+
+    ignoretests = []
+    if args.ignoretests is not None:
+        ignoretests = args.ignoretests[0].split(",")
+
+    try:
+        max_len = 0
+        max_title = ""
+        for root, dirs, files in os.walk(testpath):
+            path = root.split(os.sep)
+            for file in files:
+                if file.endswith(".yaml"):
+                    with open(os.path.join(root, file)) as f:
+                        test = yaml.full_load(f)
+                        for t in test['tests']:
+                            title = t['test_title']
+                            for s in t['stages']:
+                                if 'stage' in s:
+                                    if 'input' in s['stage']:
+                                        if 'data' in s['stage']['input']:
+                                            if len(s['stage']['input']['data']) > max_len \
+                                                and title not in ignoretests:
+                                                max_len = len(s['stage']['input']['data'])
+                                                max_title = title
+        print("Longest data: %d in test %s" % (max_len, max_title))
+    except:
+        print("Can't open files in given path!")
+        print(sys.exc_info())
+        sys.exit(1)