diff --git a/cmd/import.go b/cmd/import.go index 86ba999..4299ddc 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -446,19 +446,19 @@ func WalkFiles(afs afero.Fs, root string, rolling bool) ([]HourlyZeekLogs, []Wal // add file if it hasn't been seen before case !exists: fTracker[trimmedFileName] = fileTrack{ - lastModified: info.ModTime(), + lastModified: info.ModTime().UTC(), path: path, } // if trimmed version of the file exists in the map and the currently marked file for import // was last modified more recently than this current file, replace it with this file - case exists && fileData.lastModified.Before(info.ModTime()): + case exists && fileData.lastModified.UTC().Before(info.ModTime().UTC()): // warn the user so that this isn't a silent operation walkErrors = append(walkErrors, WalkError{Path: fTracker[trimmedFileName].path, Error: ErrSkippedDuplicateLog}) // logger.Warn().Str("original_path", fTracker[trimmedFileName].path).Str("replacement_path", path).Msg("encountered file with same name but different extension, potential duplicate log, skipping") fTracker[trimmedFileName] = fileTrack{ - lastModified: info.ModTime(), + lastModified: info.ModTime().UTC(), path: path, } // if the current file is older than the one we have already seen or no other conditions are met, skip it @@ -592,13 +592,13 @@ func GatherDailyLogs(logMap map[time.Time]HourlyZeekLogs, days []time.Time, roll slices.SortFunc(days, func(a, b time.Time) int { return a.Compare(b) }) // determine if there are any logs that are in the range to keep - today := time.Now().Truncate(24 * time.Hour) - nDaysAgo := today.Add(time.Duration(-RollingLogDaysToKeep) * 24 * time.Hour) + today := time.Now().UTC().Truncate(24 * time.Hour) + nDaysAgo := today.Add(time.Duration(-RollingLogDaysToKeep) * 24 * time.Hour).UTC() // only filter rolling! if rolling { // determine if there are any logs that are in the range to keep by checking the newest day (last in the array) - if days[len(days)-1].Compare(nDaysAgo) > -1 { + if days[len(days)-1].UTC().Compare(nDaysAgo) > -1 { shouldFilter = true } } diff --git a/cmd/import_test.go b/cmd/import_test.go index 8e41108..0ef1c65 100644 --- a/cmd/import_test.go +++ b/cmd/import_test.go @@ -1230,13 +1230,13 @@ func TestWalkFiles(t *testing.T) { require.NoError(t, err) } } else { - today := time.Now().Truncate(24 * time.Hour) + today := time.Now().UTC().Truncate(24 * time.Hour) files := []string{"conn.log", "dns.log", "http.log", "ssl.log", "open_conn.log", "open_http.log", "open_ssl.log"} switch test.name { case "Rolling Logs - Old": // test should keep all 11 days because none were in the past 2 weeks for i := -cmd.RollingLogDaysToKeep * 2; i < -cmd.RollingLogDaysToKeep; i++ { - subdirectory := today.Add(time.Duration(i) * 24 * time.Hour).Format("2006-01-02") + subdirectory := today.Add(time.Duration(i) * 24 * time.Hour).UTC().Format("2006-01-02") require.NoError(t, afs.MkdirAll(filepath.Join(test.directory, subdirectory), test.directoryPermissions)) fullPath := fmt.Sprintf("%s/%s/", test.directory, subdirectory) @@ -1249,7 +1249,7 @@ func TestWalkFiles(t *testing.T) { case "Rolling Logs - New": // test should keep only the first RollingLogDaysToKeep days for i := -cmd.RollingLogDaysToKeep - 5; i < 1; i++ { - subdirectory := today.Add(time.Duration(i) * 24 * time.Hour).Format("2006-01-02") + subdirectory := today.Add(time.Duration(i) * 24 * time.Hour).UTC().Format("2006-01-02") require.NoError(t, afs.MkdirAll(filepath.Join(test.directory, subdirectory), test.directoryPermissions)) fullPath := fmt.Sprintf("%s/%s/", test.directory, subdirectory) @@ -1258,11 +1258,10 @@ func TestWalkFiles(t *testing.T) { test.files = append(test.files, filePath) } - if i > -cmd.RollingLogDaysToKeep { + if i >= -cmd.RollingLogDaysToKeep { test.expectedFiles = append(test.expectedFiles, basicRollingHourLogs(fullPath)) } } - fmt.Println(test.files[len(test.files)-1]) } test.expectedFiles = createExpectedResults(test.expectedFiles) diff --git a/config/config.go b/config/config.go index 20ade46..b67c871 100644 --- a/config/config.go +++ b/config/config.go @@ -48,7 +48,7 @@ type ( } Env struct { // set by .env file - DBConnection string `validate:"required"` // DB_ADDRESS + DBConnection string `validate:"required,hostname_port"` // DB_ADDRESS DBUsername string `json:"-"` DBPassword string `json:"-"` HTTPExtensionsFilePath string `validate:"file"` // CONFIG_DIR/http_extensions_list.csv diff --git a/config/config_test.go b/config/config_test.go index 0786094..1b7d214 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -340,6 +340,9 @@ func TestConfig_Validate(t *testing.T) { }}, {"Env", []testCase{ {name: "DBConnection Not Host:Port", config: func(cfg *Config) { cfg.Env.DBConnection = "invalid" }, expectedErrs: []string{"'DBConnection' failed on the 'hostname_port' tag"}}, + {name: "DBConnection IP:Port", config: func(cfg *Config) { cfg.Env.DBConnection = "192.168.0.1:8123" }, expectedErrs: []string{}}, + {name: "DBConnection Docker Service:Port", config: func(cfg *Config) { cfg.Env.DBConnection = "db:8123" }, expectedErrs: []string{}}, + {name: "DBConnection localhost:Port", config: func(cfg *Config) { cfg.Env.DBConnection = "localhost:8123" }, expectedErrs: []string{}}, {name: "HTTPExtensionsFilePath Not Valid File", config: func(cfg *Config) { cfg.Env.HTTPExtensionsFilePath = "--" }, expectedErrs: []string{"'HTTPExtensionsFilePath' failed on the 'file' tag"}}, {name: "LogLevel < Min", config: func(cfg *Config) { cfg.Env.LogLevel = -1 }, expectedErrs: []string{"'LogLevel' failed on the 'min' tag"}}, {name: "LogLevel > Max", config: func(cfg *Config) { cfg.Env.LogLevel = 7 }, expectedErrs: []string{"'LogLevel' failed on the 'max' tag"}}, @@ -363,10 +366,10 @@ func TestConfig_Validate(t *testing.T) { // should never be empty after mandatory subnets are added {name: "NeverIncludedSubnets Nil", config: func(cfg *Config) { cfg.Filtering.NeverIncludedSubnets = nil }, expectedErrs: []string{"'NeverIncludedSubnets' failed on the 'required' tag"}}, {name: "NeverIncludedSubnets Empty", config: func(cfg *Config) { cfg.Filtering.NeverIncludedSubnets = []util.Subnet{} }, expectedErrs: []string{"'NeverIncludedSubnets' failed on the 'gt' tag"}}, - {name: "AlwaysIncludedDomains Not Domains", config: func(cfg *Config) { cfg.Filtering.AlwaysIncludedDomains = []string{"notadomain"} }, expectedErrs: []string{"'AlwaysIncludedDomains[0]' failed on the 'fqdn' tag"}}, - {name: "AlwaysIncludedDomains Mixed Validity", config: func(cfg *Config) { cfg.Filtering.AlwaysIncludedDomains = []string{"valid.com", "notadomain"} }, expectedErrs: []string{"'AlwaysIncludedDomains[1]' failed on the 'fqdn' tag"}}, - {name: "NeverIncludedDomains Not Domains", config: func(cfg *Config) { cfg.Filtering.NeverIncludedDomains = []string{"notadomain"} }, expectedErrs: []string{"'NeverIncludedDomains[0]' failed on the 'fqdn' tag"}}, - {name: "NeverIncludedDomains Mixed Validity", config: func(cfg *Config) { cfg.Filtering.NeverIncludedDomains = []string{"valid.com", "notadomain"} }, expectedErrs: []string{"'NeverIncludedDomains[1]' failed on the 'fqdn' tag"}}, + {name: "AlwaysIncludedDomains Not Domains", config: func(cfg *Config) { cfg.Filtering.AlwaysIncludedDomains = []string{"notadomain"} }, expectedErrs: []string{"'AlwaysIncludedDomains[0]' failed on the 'wildcard_fqdn' tag"}}, + {name: "AlwaysIncludedDomains Mixed Validity", config: func(cfg *Config) { cfg.Filtering.AlwaysIncludedDomains = []string{"valid.com", "notadomain"} }, expectedErrs: []string{"'AlwaysIncludedDomains[1]' failed on the 'wildcard_fqdn' tag"}}, + {name: "NeverIncludedDomains Not Domains", config: func(cfg *Config) { cfg.Filtering.NeverIncludedDomains = []string{"notadomain"} }, expectedErrs: []string{"'NeverIncludedDomains[0]' failed on the 'wildcard_fqdn' tag"}}, + {name: "NeverIncludedDomains Mixed Validity", config: func(cfg *Config) { cfg.Filtering.NeverIncludedDomains = []string{"valid.com", "notadomain"} }, expectedErrs: []string{"'NeverIncludedDomains[1]' failed on the 'wildcard_fqdn' tag"}}, {name: "Empty Struct", config: func(cfg *Config) { cfg.Filtering = Filtering{} }, expectedErrs: []string{"'Filtering' failed on the 'required' tag"}}, }}, {"Scoring", []testCase{ diff --git a/database/analysis_tables.go b/database/analysis_tables.go index 654c26f..a61a3b1 100644 --- a/database/analysis_tables.go +++ b/database/analysis_tables.go @@ -154,6 +154,7 @@ func (db *DB) createHistoricalFirstSeenMaterializedViews(ctx context.Context) er minSimpleState(ts) as first_seen, maxSimpleState(ts) as last_seen FROM {database:Identifier}.http + WHERE host != '' GROUP BY ( fqdn, ip) `); err != nil { return err @@ -168,6 +169,7 @@ func (db *DB) createHistoricalFirstSeenMaterializedViews(ctx context.Context) er minSimpleState(ts) as first_seen, maxSimpleState(ts) as last_seen FROM {database:Identifier}.openhttp + WHERE host != '' GROUP BY ( fqdn, ip) `); err != nil { return err diff --git a/util/subnet.go b/util/subnet.go index 704a7e4..d87184d 100644 --- a/util/subnet.go +++ b/util/subnet.go @@ -16,7 +16,7 @@ var ( ErrParseCIDRInvalidIP = fmt.Errorf("unable to parse CIDR as subnet, invalid IP address") ErrParseCIDRInvalidMask = fmt.Errorf("unable to parse CIDR as subnet, invalid mask") errParseCIDRInvalidNumMask = fmt.Errorf("unable to parse CIDR as subnet, invalid numerical value for cidr mask") - ErrIPIsNIl = fmt.Errorf("ip is nil") + ErrIPIsNil = fmt.Errorf("ip is nil") ) type Subnet struct { @@ -72,13 +72,17 @@ func (s *Subnet) UnmarshalJSON(bytes []byte) error { return nil } -// MarshalJSON marshals the Subnet struct into JSON bytes +/* +MarshalJSON marshals the Subnet struct into JSON bytes +The IP is formatted into its most human readable format. +If the CIDR mask is full/max (128 or 32), then it is not displayed. +For IPv4, the address is specifically formatted to not display IPv4 in IPv6. +*/ func (s *Subnet) MarshalJSON() ([]byte, error) { - // convert the Subnet struct to a string - // ip, err := s.ToIPString() - // if err != nil { - // return nil, err - // } + + if s.IP == nil { + return nil, ErrIPIsNil + } ip := s.IP.String() @@ -94,10 +98,14 @@ func (s *Subnet) MarshalJSON() ([]byte, error) { } -// ToIPString gets string representation of the IP address in the Subnet struct +/* +ToIPString gets string representation of the IP address in the Subnet struct +The CIDR is always included. +IPv4 is always formatted as IPv4 in IPv6. +*/ func (s *Subnet) ToIPString() (string, error) { if s.IP == nil { - return "", ErrIPIsNIl + return "", ErrIPIsNil } // verify IPv6 notation for both the ip and mask diff --git a/util/subnet_test.go b/util/subnet_test.go index 96d642b..f2fa821 100644 --- a/util/subnet_test.go +++ b/util/subnet_test.go @@ -128,30 +128,40 @@ func (s *SubnetSuite) TestNewSubnetList() { func (s *SubnetSuite) TestSubnet_UnmarshalJSON() { t := s.T() tests := []struct { - name string - input string - expected Subnet - expectedError error + name string + input string + expected Subnet + expectedMarshal string + expectedString string + expectedError error }{ { - name: "IPv4 Subnet with /24 CIDR", - input: `"::ffff:192.168.1.0/120"`, - expected: NewSubnet(&net.IPNet{IP: net.IPv4(192, 168, 1, 0).To16(), Mask: net.CIDRMask(120, 128)}), + name: "IPv4 Subnet with /24 CIDR", + input: `"::ffff:192.168.1.0/120"`, + expected: NewSubnet(&net.IPNet{IP: net.IPv4(192, 168, 1, 0).To16(), Mask: net.CIDRMask(120, 128)}), + expectedMarshal: `"192.168.1.0/24"`, + expectedString: "::ffff:192.168.1.0/120", }, { - name: "IPv6 Subnet with /64 CIDR", - input: `"2001:db8::/64"`, - expected: NewSubnet(&net.IPNet{IP: net.ParseIP("2001:db8::").To16(), Mask: net.CIDRMask(64, 128)}), + name: "IPv6 Subnet with /64 CIDR", + input: `"2001:db8::/64"`, + expected: NewSubnet(&net.IPNet{IP: net.ParseIP("2001:db8::").To16(), Mask: net.CIDRMask(64, 128)}), + expectedMarshal: `"2001:db8::/64"`, + expectedString: "2001:db8::/64", }, { - name: "IPv4 Address without CIDR", - input: `"::ffff:10.1.1.1"`, - expected: NewSubnet(&net.IPNet{IP: net.IPv4(10, 1, 1, 1).To16(), Mask: net.CIDRMask(128, 128)}), + name: "IPv4 Address without CIDR", + input: `"::ffff:10.1.1.1"`, + expected: NewSubnet(&net.IPNet{IP: net.IPv4(10, 1, 1, 1).To16(), Mask: net.CIDRMask(128, 128)}), + expectedMarshal: `"10.1.1.1"`, + expectedString: "::ffff:10.1.1.1/128", }, { - name: "IPv6 Address without CIDR", - input: `"::1"`, - expected: NewSubnet(&net.IPNet{IP: net.ParseIP("::1").To16(), Mask: net.CIDRMask(128, 128)}), + name: "IPv6 Address without CIDR", + input: `"::1"`, + expected: NewSubnet(&net.IPNet{IP: net.ParseIP("::1").To16(), Mask: net.CIDRMask(128, 128)}), + expectedMarshal: `"::1"`, + expectedString: "::1/128", }, { name: "Invalid Input", @@ -183,7 +193,7 @@ func (s *SubnetSuite) TestSubnet_UnmarshalJSON() { // marshal back to json to verify that the marshalled value matches the original input marshalled, err := subnet.MarshalJSON() require.NoError(t, err) - require.JSONEq(t, test.input, string(marshalled)) + require.JSONEq(t, test.expectedMarshal, string(marshalled)) // parse back from the newly marshalled value verify match to both the original and expected values var parsedSubnet Subnet @@ -191,6 +201,9 @@ func (s *SubnetSuite) TestSubnet_UnmarshalJSON() { require.NoError(t, err) require.Equal(t, subnet, parsedSubnet) require.Equal(t, test.expected, parsedSubnet) + + // verify ToString() value + require.Equal(t, test.expectedString, parsedSubnet.ToString()) } }) } @@ -199,35 +212,40 @@ func (s *SubnetSuite) TestSubnet_UnmarshalJSON() { func (s *SubnetSuite) TestSubnet_MarshalJSON() { t := s.T() tests := []struct { - name string - subnet Subnet - expected string - expectedError error + name string + subnet Subnet + expectedMarshal string + expectedString string + expectedError error }{ { - name: "IPv4 Subnet with /24 CIDR", - subnet: NewSubnet(&net.IPNet{IP: net.IPv4(192, 168, 1, 0).To16(), Mask: net.CIDRMask(120, 128)}), - expected: `"::ffff:192.168.1.0/120"`, + name: "IPv4 Subnet with /24 CIDR", + subnet: NewSubnet(&net.IPNet{IP: net.IPv4(192, 168, 1, 0).To16(), Mask: net.CIDRMask(120, 128)}), + expectedMarshal: `"192.168.1.0/24"`, + expectedString: "::ffff:192.168.1.0/120", }, { - name: "IPv6 Subnet with /64 CIDR", - subnet: NewSubnet(&net.IPNet{IP: net.ParseIP("2001:db8::").To16(), Mask: net.CIDRMask(64, 128)}), - expected: `"2001:db8::/64"`, + name: "IPv6 Subnet with /64 CIDR", + subnet: NewSubnet(&net.IPNet{IP: net.ParseIP("2001:db8::").To16(), Mask: net.CIDRMask(64, 128)}), + expectedMarshal: `"2001:db8::/64"`, + expectedString: "2001:db8::/64", }, { - name: "IPv4 Address without CIDR", - subnet: NewSubnet(&net.IPNet{IP: net.IPv4(10, 1, 1, 1).To16(), Mask: net.CIDRMask(128, 128)}), - expected: `"::ffff:10.1.1.1"`, + name: "IPv4 Address without CIDR", + subnet: NewSubnet(&net.IPNet{IP: net.IPv4(10, 1, 1, 1).To16(), Mask: net.CIDRMask(128, 128)}), + expectedMarshal: `"10.1.1.1"`, + expectedString: "::ffff:10.1.1.1/128", }, { - name: "IPv6 Address without CIDR", - subnet: NewSubnet(&net.IPNet{IP: net.ParseIP("::1").To16(), Mask: net.CIDRMask(128, 128)}), - expected: `"::1"`, + name: "IPv6 Address without CIDR", + subnet: NewSubnet(&net.IPNet{IP: net.ParseIP("::1").To16(), Mask: net.CIDRMask(128, 128)}), + expectedMarshal: `"::1"`, + expectedString: `"::1/128"`, }, { name: "IP is nil", subnet: NewSubnet(&net.IPNet{IP: nil, Mask: net.CIDRMask(0, 0)}), - expectedError: ErrIPIsNIl, + expectedError: ErrIPIsNil, }, } @@ -243,7 +261,7 @@ func (s *SubnetSuite) TestSubnet_MarshalJSON() { require.NoError(t, err) // verify that the marshalled value matches the expected value - require.JSONEq(t, test.expected, string(result)) + require.JSONEq(t, test.expectedMarshal, string(result)) // pass to parse subnet trimmedResult := strings.Trim(string(result), `"`) @@ -256,7 +274,7 @@ func (s *SubnetSuite) TestSubnet_MarshalJSON() { // marshal the newly parsed subnet and compare to the original marshalled result and expected value marshalledSubnet, err := parsedSubnet.MarshalJSON() require.NoError(t, err) - require.JSONEq(t, string(marshalledSubnet), test.expected) + require.JSONEq(t, string(marshalledSubnet), test.expectedMarshal) require.JSONEq(t, string(marshalledSubnet), string(result)) } }) @@ -309,7 +327,7 @@ func (s *SubnetSuite) TestSubnet_ToIPString() { { name: "Nil IP", subnet: NewSubnet(&net.IPNet{IP: nil, Mask: net.CIDRMask(0, 0)}), - expectedError: ErrIPIsNIl, + expectedError: ErrIPIsNil, }, }