From 3e365d1e45bddcb9db795a14cedbe30863c51bed Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Sun, 12 Jul 2020 20:27:49 +1000 Subject: [PATCH] HTML report generator may produce arbitrary html. Since reports are stand alone files we can allow artifacts to produce arbitrary html. This allows JS and CSS to be created by the artifacts themselves. --- .../Server/Utils/CreateCollector.yaml | 2 +- bin/report.go | 3 +- go.mod | 3 +- go.sum | 4 ++ reporting/html.go | 44 ++++++++++++++++--- services/launcher.go | 4 +- vql/filesystem/filesystem.go | 2 +- vql/tools/collector.go | 8 ++++ vql/tools/reporting.go | 22 ++++++++-- 9 files changed, 76 insertions(+), 16 deletions(-) diff --git a/artifacts/definitions/Server/Utils/CreateCollector.yaml b/artifacts/definitions/Server/Utils/CreateCollector.yaml index 1d06050059..4f02516239 100644 --- a/artifacts/definitions/Server/Utils/CreateCollector.yaml +++ b/artifacts/definitions/Server/Utils/CreateCollector.yaml @@ -210,7 +210,7 @@ sources: ) LET definitions <= SELECT * FROM chain( - a = { SELECT name, parameters, sources, reports + a = { SELECT name, description, parameters, sources, reports FROM artifact_definitions(names=Artifacts) WHERE name =~ "^(Custom|Packs)\\." AND log(message="Adding artifact_definition for " + name) }, diff --git a/bin/report.go b/bin/report.go index 2c7675b86a..c5fa3ccc8e 100644 --- a/bin/report.go +++ b/bin/report.go @@ -45,7 +45,8 @@ func doHTMLReport() { for _, artifact_name := range result.Context.Request.Artifacts { template_engine, err := reporting.NewHTMLTemplateEngine( config_obj, context.Background(), nil, /* default scope */ - vql_subsystem.NullACLManager{}, repository, artifact_name) + vql_subsystem.NullACLManager{}, repository, artifact_name, + false /* sanitize_html */) kingpin.FatalIfError(err, "Generating report") template_engine.SetEnv("ClientId", *report_command_flow_client) diff --git a/go.mod b/go.mod index c608f8a895..20ce21249a 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,7 @@ require ( golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae + golang.org/x/text v0.3.3 // indirect golang.org/x/tools v0.0.0-20200207131002-533eb2654509 // indirect google.golang.org/api v0.11.0 google.golang.org/appengine v1.6.5 // indirect @@ -119,7 +120,7 @@ require ( www.velocidex.com/golang/go-prefetch v0.0.0-20190703150313-0469fa2f85cf www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 - www.velocidex.com/golang/vfilter v0.0.0-20200605101505-2a4beeea86e2 + www.velocidex.com/golang/vfilter v0.0.0-20200711150936-76e1310efce7 www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b ) diff --git a/go.sum b/go.sum index d80fbcaca0..a670694075 100644 --- a/go.sum +++ b/go.sum @@ -493,6 +493,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -611,5 +613,7 @@ www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 h1:XqZddiA www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500/go.mod h1:DVzloLH8L+oF3zma1Jisaat5bGF+4VLggDcYlIp00ns= www.velocidex.com/golang/vfilter v0.0.0-20200605101505-2a4beeea86e2 h1:iyKy7xjJL+71/kL5R4KrWJE6Keq++eULTTn9pTQc3SM= www.velocidex.com/golang/vfilter v0.0.0-20200605101505-2a4beeea86e2/go.mod h1:mABF6rGkfq9qwvo2SppBxYonhnd8OPSA6rxzzl75A4Y= +www.velocidex.com/golang/vfilter v0.0.0-20200711150936-76e1310efce7 h1:0p7AX5nmluXkjJ3AEMN8YeqgMkYU/qiVXt325PBFwa4= +www.velocidex.com/golang/vfilter v0.0.0-20200711150936-76e1310efce7/go.mod h1:mABF6rGkfq9qwvo2SppBxYonhnd8OPSA6rxzzl75A4Y= www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b h1:z5v5o1dhtzaxvlWm6qSTYZ4OTr56Ol2JpM1Y5Wu9zQE= www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b/go.mod h1:tXxIx8UJuI81Hoxcv0DTq2a1Pi1H6l1uCf4dhqUSUkw= diff --git a/reporting/html.go b/reporting/html.go index 7619b7deb0..0dbd9d6b7c 100644 --- a/reporting/html.go +++ b/reporting/html.go @@ -26,10 +26,11 @@ import ( type HTMLTemplateEngine struct { *BaseTemplateEngine - tmpl *template.Template - ctx context.Context - log_writer *logWriter - Data map[string]*actions_proto.VQLResponse + tmpl *template.Template + ctx context.Context + log_writer *logWriter + Data map[string]*actions_proto.VQLResponse + SanitizeHTML bool } func (self *HTMLTemplateEngine) Expand(values ...interface{}) interface{} { @@ -65,10 +66,35 @@ func (self *HTMLTemplateEngine) Table(values ...interface{}) interface{} { default: return t + case []*ordereddict.Dict: + columns := []string{} + + result := "\n" + + for _, item := range t { + if len(columns) == 0 { + columns = item.Keys() + result += " \n" + for _, name := range columns { + result += " \n" + } + result += " \n" + } + + result += " \n" + for _, name := range columns { + value, _ := item.Get(name) + result += fmt.Sprintf(" \n", value) + } + result += " \n" + } + result += "
" + name + "
%v
\n" + return result + case chan *ordereddict.Dict: columns := []string{} - result := "\n" + result := "
\n" for item := range t { if len(columns) == 0 { @@ -141,6 +167,10 @@ func (self *HTMLTemplateEngine) Execute(template_string string) (string, error) output_string := strings.ReplaceAll(string(output), "
", "
") + if !self.SanitizeHTML { + return output_string, err + } + // Sanitize the HTML. return bm_policy.Sanitize(output_string), nil } @@ -213,7 +243,8 @@ func NewHTMLTemplateEngine( scope *vfilter.Scope, acl_manager vql_subsystem.ACLManager, repository *artifacts.Repository, - artifact_name string) ( + artifact_name string, + sanitize_html bool) ( *HTMLTemplateEngine, error) { base_engine, err := newBaseTemplateEngine( @@ -229,6 +260,7 @@ func NewHTMLTemplateEngine( BaseTemplateEngine: base_engine, ctx: ctx, log_writer: log_writer, + SanitizeHTML: sanitize_html, } template_engine.tmpl = template.New("").Funcs(sprig.TxtFuncMap()).Funcs( template.FuncMap{ diff --git a/services/launcher.go b/services/launcher.go index 1f61b50c1b..abc4762640 100644 --- a/services/launcher.go +++ b/services/launcher.go @@ -64,7 +64,7 @@ func CompileCollectorArgs( return nil, err } - ensureToolsDeclared(ctx, config_obj, artifact) + EnsureToolsDeclared(ctx, config_obj, artifact) } // Add any artifact dependencies. @@ -106,7 +106,7 @@ func getDependentTools( } // Make sure we know about tools the artifact itself defines. -func ensureToolsDeclared( +func EnsureToolsDeclared( ctx context.Context, config_obj *config_proto.Config, artifact *artifacts_proto.Artifact) error { diff --git a/vql/filesystem/filesystem.go b/vql/filesystem/filesystem.go index ebf3a3e2aa..5cc5bfdfd5 100644 --- a/vql/filesystem/filesystem.go +++ b/vql/filesystem/filesystem.go @@ -293,7 +293,7 @@ func (self ReadFileFunction) Info(scope *vfilter.Scope, type_map *vfilter.TypeMa return &vfilter.FunctionInfo{ Name: "read_file", Doc: "Read a file into a string.", - ArgType: type_map.AddType(scope, &ReadFileArgs{}), + ArgType: type_map.AddType(scope, &ReadFileFunctionArgs{}), } } diff --git a/vql/tools/collector.go b/vql/tools/collector.go index 1763b8bef9..b6e0139d3a 100644 --- a/vql/tools/collector.go +++ b/vql/tools/collector.go @@ -15,6 +15,7 @@ import ( "www.velocidex.com/golang/velociraptor/config" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/reporting" + "www.velocidex.com/golang/velociraptor/services" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" "www.velocidex.com/golang/vfilter" ) @@ -150,6 +151,13 @@ func (self CollectPlugin) Call( continue } + + err = services.EnsureToolsDeclared(ctx, config_obj, artifact) + if err != nil { + scope.Log("collect: ", name) + continue + } + artifact_definitions = append(artifact_definitions, artifact) request := &actions_proto.VQLCollectorArgs{} diff --git a/vql/tools/reporting.go b/vql/tools/reporting.go index 97fe7359d3..b985e2c4e2 100644 --- a/vql/tools/reporting.go +++ b/vql/tools/reporting.go @@ -18,6 +18,11 @@ import ( "www.velocidex.com/golang/vfilter" ) +type ReportPart struct { + Artifact *artifacts_proto.Artifact + HTML string +} + func getHTMLTemplate(name string, repository *artifacts.Repository) (string, error) { template_artifact, ok := repository.Get(name) if !ok || len(template_artifact.Reports) == 0 { @@ -60,17 +65,22 @@ func produceReport( return err } - content_writer := &bytes.Buffer{} + parts := []*ReportPart{} + main := "" for _, definition := range definitions { + content_writer := &bytes.Buffer{} for _, report := range definition.Reports { if report.Type != "client" { continue } + // Do not sanitize_html since we are writing a + // stand along HTML file - artifacts may + // generate arbitrary HTML. template_engine, err := reporting.NewHTMLTemplateEngine( config_obj, context.Background(), subscope, vql_subsystem.NullACLManager{}, repository, - definition.Name) + definition.Name, false /* sanitize_html */) if err != nil { return err } @@ -87,17 +97,21 @@ func produceReport( content_writer.Write([]byte(res)) } + parts = append(parts, &ReportPart{ + Artifact: definition, HTML: content_writer.String()}) + main += content_writer.String() } template_engine, err := reporting.NewHTMLTemplateEngine( config_obj, context.Background(), subscope, vql_subsystem.NullACLManager{}, repository, - template) + template, false /* sanitize_html */) if err != nil { return err } - template_engine.SetEnv("main", content_writer.String()) + template_engine.SetEnv("main", main) + template_engine.SetEnv("parts", parts) result, err := template_engine.RenderRaw( html_template_string, template_engine.Env.ToDict())