8000 Fix tests by caffeinatedpixel · Pull Request #49 · activecm/rita · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Fix tests #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an i 8000 ssue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 19, 2025
Merged

Fix tests #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c540977
Fix tests
caffeinatedpixel May 19, 2025
64bb1f5
Update WalkFiles to use UTC
caffeinatedpixel May 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions cmd/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,19 +446,19 @@ func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []Wal
// add file if it hasn't been seen before
case !exists:
fTracker[trimmedFileName] = fileTrack{
lastModified: info.ModTime(),
lastModified: info.ModTime().UTC(),
path: path,
}
// if trimmed version of the file exists in the map and the currently marked file for import
// was last modified more recently than this current file, replace it with this file
case exists && fileData.lastModified.Before(info.ModTime()):
case exists && fileData.lastModified.UTC().Before(info.ModTime().UTC()):

// warn the user so that this isn't a silent operation
walkErrors = append(walkErrors, WalkError{Path: fTracker[trimmedFileName].path, Error: ErrSkippedDuplicateLog})
// logger.Warn().Str("original_path", fTracker[trimmedFileName].path).Str("replacement_path", path).Msg("encountered file with same name but different extension, potential duplicate log, skipping")

fTracker[trimmedFileName] = fileTrack{
lastModified: info.ModTime(),
lastModified: info.ModTime().UTC(),
path: path,
}
// if the current file is older than the one we have already seen or no other conditions are met, skip it
Expand Down Expand Up @@ -592,13 +592,13 @@ func GatherDailyLogs(logMap map[time.Time]HourlyZeekLogs, days []time.Time, roll
slices.SortFunc(days, func(a, b time.Time) int { return a.Compare(b) })

// determine if there are any logs that are in the range to keep
today := time.Now().Truncate(24 * time.Hour)
nDaysAgo := today.Add(time.Duration(-RollingLogDaysToKeep) * 24 * time.Hour)
today := time.Now().UTC().Truncate(24 * time.Hour)
nDaysAgo := today.Add(time.Duration(-RollingLogDaysToKeep) * 24 * time.Hour).UTC()

// only filter rolling!
if rolling {
// determine if there are any logs that are in the range to keep by checking the newest day (last in the array)
if days[len(days)-1].Compare(nDaysAgo) > -1 {
if days[len(days)-1].UTC().Compare(nDaysAgo) > -1 {
shouldFilter = true
}
}
Expand Down
9 changes: 4 additions & 5 deletions cmd/import_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1230,13 +1230,13 @@ func TestWalkFiles(t *testing.T) {
require.NoError(t, err)
}
} else {
today := time.Now().Truncate(24 * time.Hour)
today := time.Now().UTC().Truncate(24 * time.Hour)
files := []string{"conn.log", "dns.log", "http.log", "ssl.log", "open_conn.log", "open_http.log", "open_ssl.log"}
switch test.name {
case "Rolling Logs - Old":
// test should keep all 11 days because none were in the past 2 weeks
for i := -cmd.RollingLogDaysToKeep * 2; i < -cmd.RollingLogDaysToKeep; i++ {
subdirectory := today.Add(time.Duration(i) * 24 * time.Hour).Format("2006-01-02")
subdirectory := today.Add(time.Duration(i) * 24 * time.Hour).UTC().Format("2006-01-02")
require.NoError(t, afs.MkdirAll(filepath.Join(test.directory, subdirectory), test.directoryPermissions))
fullPath := fmt.Sprintf("%s/%s/", test.directory, subdirectory)

Expand All @@ -1249,7 +1249,7 @@ func TestWalkFiles(t *testing.T) {
case "Rolling Logs - New":
// test should keep only the first RollingLogDaysToKeep days
for i := -cmd.RollingLogDaysToKeep - 5; i < 1; i++ {
subdirectory := today.Add(time.Duration(i) * 24 * time.Hour).Format("2006-01-02")
subdirectory := today.Add(time.Duration(i) * 24 * time.Hour).UTC().Format("2006-01-02")
require.NoError(t, afs.MkdirAll(filepath.Join(test.directory, subdirectory), test.directoryPermissions))
fullPath := fmt.Sprintf("%s/%s/", test.directory, subdirectory)

Expand All @@ -1258,11 +1258,10 @@ func TestWalkFiles(t *testing.T) {
test.files = append(test.files, filePath)
}

if i > -cmd.RollingLogDaysToKeep {
if i >= -cmd.RollingLogDaysToKeep {
test.expectedFiles = append(test.expectedFiles, basicRollingHourLogs(fullPath))
}
}
fmt.Println(test.files[len(test.files)-1])

}
test.expectedFiles = createExpectedResults(test.expectedFiles)
Expand Down
2 changes: 1 addition & 1 deletion config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ type (
}

Env struct { // set by .env file
DBConnection string `validate:"required"` // DB_ADDRESS
DBConnection string `validate:"required,hostname_port"` // DB_ADDRESS
DBUsername string `json:"-"`
DBPassword string `json:"-"`
HTTPExtensionsFilePath string `validate:"file"` // CONFIG_DIR/http_extensions_list.csv
Expand Down
11 changes: 7 additions & 4 deletions config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,9 @@ func TestConfig_Validate(t *testing.T) {
}},
{"Env", []testCase{
{name: "DBConnection Not Host:Port", config: func(cfg *Config) { cfg.Env.DBConnection = "invalid" }, expectedErrs: []string{"'DBConnection' failed on the 'hostname_port' tag"}},
{name: "DBConnection IP:Port", config: func(cfg *Config) { cfg.Env.DBConnection = "192.168.0.1:8123" }, expectedErrs: []string{}},
{name: "DBConnection Docker Service:Port", config: func(cfg *Config) { cfg.Env.DBConnection = "db:8123" }, expectedErrs: []string{}},
{name: "DBConnection localhost:Port", config: func(cfg *Config) { cfg.Env.DBConnection = "localhost:8123" }, expectedErrs: []string{}},
{name: "HTTPExtensionsFilePath Not Valid File", config: func(cfg *Config) { cfg.Env.HTTPExtensionsFilePath = "--" }, expectedErrs: []string{"'HTTPExtensionsFilePath' failed on the 'file' tag"}},
{name: "LogLevel < Min", config: func(cfg *Config) { cfg.Env.LogLevel = -1 }, expectedErrs: []string{"'LogLevel' failed on the 'min' tag"}},
{name: "LogLevel > Max", config: func(cfg *Config) { cfg.Env.LogLevel = 7 }, expectedErrs: []string{"'LogLevel' failed on the 'max' tag"}},
Expand All @@ -363,10 +366,10 @@ func TestConfig_Validate(t *testing.T) {
// should never be empty after mandatory subnets are added
{name: "NeverIncludedSubnets Nil", config: func(cfg *Config) { cfg.Filtering.NeverIncludedSubnets = nil }, expectedErrs: []string{"'NeverIncludedSubnets' failed on the 'required' tag"}},
{name: "NeverIncludedSubnets Empty", config: func(cfg *Config) { cfg.Filtering.NeverIncludedSubnets = []util.Subnet{} }, expectedErrs: []string{"'NeverIncludedSubnets' failed on the 'gt' tag"}},
{name: "AlwaysIncludedDomains Not Domains", config: func(cfg *Config) { cfg.Filtering.AlwaysIncludedDomains = []string{"notadomain"} }, expectedErrs: []string{"'AlwaysIncludedDomains[0]' failed on the 'fqdn' tag"}},
{name: "AlwaysIncludedDomains Mixed Validity", config: func(cfg *Config) { cfg.Filtering.AlwaysIncludedDomains = []string{"valid.com", "notadomain"} }, expectedErrs: []string{"'AlwaysIncludedDomains[1]' failed on the 'fqdn' tag"}},
{name: "NeverIncludedDomains Not Domains", config: func(cfg *Config) { cfg.Filtering.NeverIncludedDomains = []string{"notadomain"} }, expectedErrs: []string{"'NeverIncludedDomains[0]' failed on the 'fqdn' tag"}},
{name: "NeverIncludedDomains Mixed Validity", config: func(cfg *Config) { cfg.Filtering.NeverIncludedDomains = []string{"valid.com", "notadomain"} }, expectedErrs: []string{"'NeverIncludedDomains[1]' failed on the 'fqdn' tag"}},
{name: "AlwaysIncludedDomains Not Domains", config: func(cfg *Config) { cfg.Filtering.AlwaysIncludedDomains = []string{"notadomain"} }, expectedErrs: []string{"'AlwaysIncludedDomains[0]' failed on the 'wildcard_fqdn' tag"}},
{name: "AlwaysIncludedDomains Mixed Validity", config: func(cfg *Config) { cfg.Filtering.AlwaysIncludedDomains = []string{"valid.com", "notadomain"} }, expectedErrs: []string{"'AlwaysIncludedDomains[1]' failed on the 'wildcard_fqdn' tag"}},
{name: "NeverIncludedDomains Not Domains", config: func(cfg *Config) { cfg.Filtering.NeverIncludedDomains = []string{"notadomain"} }, expectedErrs: []string{"'NeverIncludedDomains[0]' failed on the 'wildcard_fqdn' tag"}},
{name: "NeverIncludedDomains Mixed Validity", config: func(cfg *Config) { cfg.Filtering.NeverIncludedDomains = []string{"valid.com", "notadomain"} }, expectedErrs: []string{"'NeverIncludedDomains[1]' failed on the 'wildcard_fqdn' tag"}},
{name: "Empty Struct", config: func(cfg *Config) { cfg.Filtering = Filtering{} }, expectedErrs: []string{"'Filtering' failed on the 'required' tag"}},
}},
{"Scoring", []testCase{
Expand Down
2 changes: 2 additions & 0 deletions database/analysis_tables.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ func (db *DB) createHistoricalFirstSeenMaterializedViews(ctx context.Context) er
minSimpleState(ts) as first_seen,
maxSimpleState(ts) as last_seen
FROM {database:Identifier}.http
WHERE host != ''
GROUP BY ( fqdn, ip)
`); err != nil {
return err
Expand All @@ -168,6 +169,7 @@ func (db *DB) createHistoricalFirstSeenMaterializedViews(ctx context.Context) er
minSimpleState(ts) as first_seen,
maxSimpleState(ts) as last_seen
FROM {database:Identifier}.openhttp
WHERE host != ''
GROUP BY ( fqdn, ip)
`); err != nil {
return err
Expand Down
26 changes: 17 additions & 9 deletions util/subnet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ var (
ErrParseCIDRInvalidIP = fmt.Errorf("unable to parse CIDR as subnet, invalid IP address")
ErrParseCIDRInvalidMask = fmt.Errorf("unable to parse CIDR as subnet, invalid mask")
errParseCIDRInvalidNumMask = fmt.Errorf("unable to parse CIDR as subnet, invalid numerical value for cidr mask")
ErrIPIsNIl = fmt.Errorf("ip is nil")
ErrIPIsNil = fmt.Errorf("ip is nil")
)

type Subnet struct {
Expand Down Expand Up @@ -72,13 +72,17 @@ func (s *Subnet) UnmarshalJSON(bytes []byte) error {
return nil
}

// MarshalJSON marshals the Subnet struct into JSON bytes
/*
MarshalJSON marshals the Subnet struct into JSON bytes
The IP is formatted into its most human readable format.
If the CIDR mask is full/max (128 or 32), then it is not displayed.
For IPv4, the address is specifically formatted to not display IPv4 in IPv6.
*/
func (s *Subnet) MarshalJSON() ([]byte, error) {
// convert the Subnet struct to a string
// ip, err := s.ToIPString()
// if err != nil {
// return nil, err
// }

if s.IP == nil {
return nil, ErrIPIsNil
}

ip := s.IP.String()

Expand All @@ -94,10 +98,14 @@ func (s *Subnet) MarshalJSON() ([]byte, error) {

}

// ToIPString gets string representation of the IP address in the Subnet struct
/*
ToIPString gets string representation of the IP address in the Subnet struct
The CIDR is always included.
IPv4 is always formatted as IPv4 in IPv6.
*/
func (s *Subnet) ToIPString() (string, error) {
if s.IP == nil {
return "", ErrIPIsNIl
return "", ErrIPIsNil
}

// verify IPv6 notation for both the ip and mask
Expand Down
92 changes: 55 additions & 37 deletions util/subnet_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,30 +128,40 @@ func (s *SubnetSuite) TestNewSubnetList() {
func (s *SubnetSuite) TestSubnet_UnmarshalJSON() {
t := s.T()
tests := []struct {
name string
input string
expected Subnet
expectedError error
name string
input string
expected Subnet
expectedMarshal string
expectedString string
expectedError error
}{
{
name: "IPv4 Subnet with /24 CIDR",
input: `"::ffff:192.168.1.0/120"`,
expected: NewSubnet(&net.IPNet{IP: net.IPv4(192, 168, 1, 0).To16(), Mask: net.CIDRMask(120, 128)}),
name: "IPv4 Subnet with /24 CIDR",
input: `"::ffff:192.168.1.0/120"`,
expected: NewSubnet(&net.IPNet{IP: net.IPv4(192, 168, 1, 0).To16(), Mask: net.CIDRMask(120, 128)}),
expectedMarshal: `"192.168.1.0/24"`,
expectedString: "::ffff:192.168.1.0/120",
},
{
name: "IPv6 Subnet with /64 CIDR",
input: `"2001:db8::/64"`,
expected: NewSubnet(&net.IPNet{IP: net.ParseIP("2001:db8::").To16(), Mask: net.CIDRMask(64, 128)}),
name: "IPv6 Subnet with /64 CIDR",
input: `"2001:db8::/64"`,
expected: NewSubnet(&net.IPNet{IP: net.ParseIP("2001:db8::").To16(), Mask: net.CIDRMask(64, 128)}),
expectedMarshal: `"2001:db8::/64"`,
expectedString: "2001:db8::/64",
},
{
name: "IPv4 Address without CIDR",
input: `"::ffff:10.1.1.1"`,
expected: NewSubnet(&net.IPNet{IP: net.IPv4(10, 1, 1, 1).To16(), Mask: net.CIDRMask(128, 128)}),
name: "IPv4 Address without CIDR",
input: `"::ffff:10.1.1.1"`,
expected: NewSubnet(&net.IPNet{IP: net.IPv4(10, 1, 1, 1).To16(), Mask: net.CIDRMask(128, 128)}),
expectedMarshal: `"10.1.1.1"`,
expectedString: "::ffff:10.1.1.1/128",
},
{
name: "IPv6 Address without CIDR",
input: `"::1"`,
expected: NewSubnet(&net.IPNet{IP: net.ParseIP("::1").To16(), Mask: net.CIDRMask(128, 128)}),
name: "IPv6 Address without CIDR",
input: `"::1"`,
expected: NewSubnet(&net.IPNet{IP: net.ParseIP("::1").To16(), Mask: net.CIDRMask(128, 128)}),
expectedMarshal: `"::1"`,
expectedString: "::1/128",
},
{
name: "Invalid Input",
Expand Down Expand Up @@ -183,14 +193,17 @@ func (s *SubnetSuite) TestSubnet_UnmarshalJSON() {
// marshal back to json to verify that the marshalled value matches the original input
marshalled, err := subnet.MarshalJSON()
require.NoError(t, err)
require.JSONEq(t, test.input, string(marshalled))
require.JSONEq(t, test.expectedMarshal, string(marshalled))

// parse back from the newly marshalled value verify match to both the original and expected values
var parsedSubnet Subnet
err = parsedSubnet.UnmarshalJSON(marshalled)
require.NoError(t, err)
require.Equal(t, subnet, parsedSubnet)
require.Equal(t, test.expected, parsedSubnet)

// verify ToString() value
require.Equal(t, test.expectedString, parsedSubnet.ToString())
}
})
}
Expand All @@ -199,35 +212,40 @@ func (s *SubnetSuite) TestSubnet_UnmarshalJSON() {
func (s *SubnetSuite) TestSubnet_MarshalJSON() {
t := s.T()
tests := []struct {
name string
subnet Subnet
expected string
expectedError error
name string
subnet Subnet
expectedMarshal string
expectedString string
expectedError error
}{
{
name: "IPv4 Subnet with /24 CIDR",
subnet: NewSubnet(&net.IPNet{IP: net.IPv4(192, 168, 1, 0).To16(), Mask: net.CIDRMask(120, 128)}),
expected: `"::ffff:192.168.1.0/120"`,
name: "IPv4 Subnet with /24 CIDR",
subnet: NewSubnet(&net.IPNet{IP: net.IPv4(192, 168, 1, 0).To16(), Mask: net.CIDRMask(120, 128)}),
expectedMarshal: `"192.168.1.0/24"`,
expectedString: "::ffff:192.168.1.0/120",
},
{
name: "IPv6 Subnet with /64 CIDR",
subnet: NewSubnet(&net.IPNet{IP: net.ParseIP("2001:db8::").To16(), Mask: net.CIDRMask(64, 128)}),
expected: `"2001:db8::/64"`,
name: "IPv6 Subnet with /64 CIDR",
subnet: NewSubnet(&net.IPNet{IP: net.ParseIP("2001:db8::").To16(), Mask: net.CIDRMask(64, 128)}),
expectedMarshal: `"2001:db8::/64"`,
expectedString: "2001:db8::/64",
},
{
name: "IPv4 Address without CIDR",
subnet: NewSubnet(&net.IPNet{IP: net.IPv4(10, 1, 1, 1).To16(), Mask: net.CIDRMask(128, 128)}),
expected: `"::ffff:10.1.1.1"`,
name: "IPv4 Address without CIDR",
subnet: NewSubnet(&net.IPNet{IP: net.IPv4(10, 1, 1, 1).To16(), Mask: net.CIDRMask(128, 128)}),
expectedMarshal: `"10.1.1.1"`,
expectedString: "::ffff:10.1.1.1/128",
},
{
name: "IPv6 Address without CIDR",
subnet: NewSubnet(&net.IPNet{IP: net.ParseIP("::1").To16(), Mask: net.CIDRMask(128, 128)}),
expected: `"::1"`,
name: "IPv6 Address without CIDR",
subnet: NewSubnet(&net.IPNet{IP: net.ParseIP("::1").To16(), Mask: net.CIDRMask(128, 128)}),
expectedMarshal: `"::1"`,
expectedString: `"::1/128"`,
},
{
name: "IP is nil",
subnet: NewSubnet(&net.IPNet{IP: nil, Mask: net.CIDRMask(0, 0)}),
expectedError: ErrIPIsNIl,
expectedError: ErrIPIsNil,
},
}

Expand All @@ -243,7 +261,7 @@ func (s *SubnetSuite) TestSubnet_MarshalJSON() {
require.NoError(t, err)

// verify that the marshalled value matches the expected value
require.JSONEq(t, test.expected, string(result))
require.JSONEq(t, test.expectedMarshal, string(result))

// pass to parse subnet
trimmedResult := strings.Trim(string(result), `"`)
Expand All @@ -256,7 +274,7 @@ func (s *SubnetSuite) TestSubnet_MarshalJSON() {
// marshal the newly parsed subnet and compare to the original marshalled result and expected value
marshalledSubnet, err := parsedSubnet.MarshalJSON()
require.NoError(t, err)
require.JSONEq(t, string(marshalledSubnet), test.expected)
require.JSONEq(t, string(marshalledSubnet), test.expectedMarshal)
require.JSONEq(t, string(marshalledSubnet), string(result))
}
})
Expand Down Expand Up @@ -309,7 +327,7 @@ func (s *SubnetSuite) TestSubnet_ToIPString() {
{
name: "Nil IP",
subnet: NewSubnet(&net.IPNet{IP: nil, Mask: net.CIDRMask(0, 0)}),
expectedError: ErrIPIsNIl,
expectedError: ErrIPIsNil,
},
}

Expand Down
0