Velocidex · scudette · Jul 12, 2020 · Jul 12, 2020
diff --git a/artifacts/definitions/Server/Utils/CreateCollector.yaml b/artifacts/definitions/Server/Utils/CreateCollector.yaml
@@ -210,7 +210,7 @@ sources:
       )
 
       LET definitions <= SELECT * FROM chain(
-      a = { SELECT name, parameters, sources, reports
+      a = { SELECT name, description, parameters, sources, reports
             FROM artifact_definitions(names=Artifacts)
             WHERE name =~ "^(Custom|Packs)\\." AND
               log(message="Adding artifact_definition for " + name) },

diff --git a/bin/report.go b/bin/report.go
@@ -45,7 +45,8 @@ func doHTMLReport() {
 	for _, artifact_name := range result.Context.Request.Artifacts {
 		template_engine, err := reporting.NewHTMLTemplateEngine(
 			config_obj, context.Background(), nil, /* default scope */
-			vql_subsystem.NullACLManager{}, repository, artifact_name)
+			vql_subsystem.NullACLManager{}, repository, artifact_name,
+			false /* sanitize_html */)
 		kingpin.FatalIfError(err, "Generating report")
 
 		template_engine.SetEnv("ClientId", *report_command_flow_client)

diff --git a/go.mod b/go.mod
@@ -98,6 +98,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
 	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
+	golang.org/x/text v0.3.3 // indirect
 	golang.org/x/tools v0.0.0-20200207131002-533eb2654509 // indirect
 	google.golang.org/api v0.11.0
 	google.golang.org/appengine v1.6.5 // indirect
@@ -119,7 +120,7 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20190703150313-0469fa2f85cf
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196
 	www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500
-	www.velocidex.com/golang/vfilter v0.0.0-20200605101505-2a4beeea86e2
+	www.velocidex.com/golang/vfilter v0.0.0-20200711150936-76e1310efce7
 	www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b
 )
 

diff --git a/go.sum b/go.sum
@@ -493,6 +493,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -611,5 +613,7 @@ www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 h1:XqZddiA
 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500/go.mod h1:DVzloLH8L+oF3zma1Jisaat5bGF+4VLggDcYlIp00ns=
 www.velocidex.com/golang/vfilter v0.0.0-20200605101505-2a4beeea86e2 h1:iyKy7xjJL+71/kL5R4KrWJE6Keq++eULTTn9pTQc3SM=
 www.velocidex.com/golang/vfilter v0.0.0-20200605101505-2a4beeea86e2/go.mod h1:mABF6rGkfq9qwvo2SppBxYonhnd8OPSA6rxzzl75A4Y=
+www.velocidex.com/golang/vfilter v0.0.0-20200711150936-76e1310efce7 h1:0p7AX5nmluXkjJ3AEMN8YeqgMkYU/qiVXt325PBFwa4=
+www.velocidex.com/golang/vfilter v0.0.0-20200711150936-76e1310efce7/go.mod h1:mABF6rGkfq9qwvo2SppBxYonhnd8OPSA6rxzzl75A4Y=
 www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b h1:z5v5o1dhtzaxvlWm6qSTYZ4OTr56Ol2JpM1Y5Wu9zQE=
 www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b/go.mod h1:tXxIx8UJuI81Hoxcv0DTq2a1Pi1H6l1uCf4dhqUSUkw=
diff --git a/reporting/html.go b/reporting/html.go
@@ -26,10 +26,11 @@ import (
 
 type HTMLTemplateEngine struct {
 	*BaseTemplateEngine
-	tmpl       *template.Template
-	ctx        context.Context
-	log_writer *logWriter
-	Data       map[string]*actions_proto.VQLResponse
+	tmpl         *template.Template
+	ctx          context.Context
+	log_writer   *logWriter
+	Data         map[string]*actions_proto.VQLResponse
+	SanitizeHTML bool
 }
 
 func (self *HTMLTemplateEngine) Expand(values ...interface{}) interface{} {
@@ -65,10 +66,35 @@ func (self *HTMLTemplateEngine) Table(values ...interface{}) interface{} {
 	default:
 		return t
 
+	case []*ordereddict.Dict:
+		columns := []string{}
+
+		result := "<table class=\"table table-striped\">\n"
+
+		for _, item := range t {
+			if len(columns) == 0 {
+				columns = item.Keys()
+				result += "  <tr>\n"
+				for _, name := range columns {
+					result += "    <th>" + name + "</th>\n"
+				}
+				result += "  </tr>\n"
+			}
+
+			result += "  <tr>\n"
+			for _, name := range columns {
+				value, _ := item.Get(name)
+				result += fmt.Sprintf("    <td>%v</td>\n", value)
+			}
+			result += "  </tr>\n"
+		}
+		result += "</table>\n"
+		return result
+
 	case chan *ordereddict.Dict:
 		columns := []string{}
 
-		result := "<table>\n"
+		result := "<table class=\"table table-striped\">\n"
 
 		for item := range t {
 			if len(columns) == 0 {
@@ -141,6 +167,10 @@ func (self *HTMLTemplateEngine) Execute(template_string string) (string, error)
 	output_string := strings.ReplaceAll(string(output),
 		"<table>", "<table class=\"table table-striped\">")
 
+	if !self.SanitizeHTML {
+		return output_string, err
+	}
+
 	// Sanitize the HTML.
 	return bm_policy.Sanitize(output_string), nil
 }
@@ -213,7 +243,8 @@ func NewHTMLTemplateEngine(
 	scope *vfilter.Scope,
 	acl_manager vql_subsystem.ACLManager,
 	repository *artifacts.Repository,
-	artifact_name string) (
+	artifact_name string,
+	sanitize_html bool) (
 	*HTMLTemplateEngine, error) {
 
 	base_engine, err := newBaseTemplateEngine(
@@ -229,6 +260,7 @@ func NewHTMLTemplateEngine(
 		BaseTemplateEngine: base_engine,
 		ctx:                ctx,
 		log_writer:         log_writer,
+		SanitizeHTML:       sanitize_html,
 	}
 	template_engine.tmpl = template.New("").Funcs(sprig.TxtFuncMap()).Funcs(
 		template.FuncMap{

diff --git a/services/launcher.go b/services/launcher.go
@@ -64,7 +64,7 @@ func CompileCollectorArgs(
 			return nil, err
 		}
 
-		ensureToolsDeclared(ctx, config_obj, artifact)
+		EnsureToolsDeclared(ctx, config_obj, artifact)
 	}
 
 	// Add any artifact dependencies.
@@ -106,7 +106,7 @@ func getDependentTools(
 }
 
 // Make sure we know about tools the artifact itself defines.
-func ensureToolsDeclared(
+func EnsureToolsDeclared(
 	ctx context.Context, config_obj *config_proto.Config,
 	artifact *artifacts_proto.Artifact) error {
 

diff --git a/vql/filesystem/filesystem.go b/vql/filesystem/filesystem.go
@@ -293,7 +293,7 @@ func (self ReadFileFunction) Info(scope *vfilter.Scope, type_map *vfilter.TypeMa
 	return &vfilter.FunctionInfo{
 		Name:    "read_file",
 		Doc:     "Read a file into a string.",
-		ArgType: type_map.AddType(scope, &ReadFileArgs{}),
+		ArgType: type_map.AddType(scope, &ReadFileFunctionArgs{}),
 	}
 }
 

diff --git a/vql/tools/collector.go b/vql/tools/collector.go
@@ -15,6 +15,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/config"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/reporting"
+	"www.velocidex.com/golang/velociraptor/services"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
 )
@@ -150,6 +151,13 @@ func (self CollectPlugin) Call(
 				continue
 
 			}
+
+			err = services.EnsureToolsDeclared(ctx, config_obj, artifact)
+			if err != nil {
+				scope.Log("collect: ", name)
+				continue
+			}
+
 			artifact_definitions = append(artifact_definitions, artifact)
 
 			request := &actions_proto.VQLCollectorArgs{}

diff --git a/vql/tools/reporting.go b/vql/tools/reporting.go
@@ -18,6 +18,11 @@ import (
 	"www.velocidex.com/golang/vfilter"
 )
 
+type ReportPart struct {
+	Artifact *artifacts_proto.Artifact
+	HTML     string
+}
+
 func getHTMLTemplate(name string, repository *artifacts.Repository) (string, error) {
 	template_artifact, ok := repository.Get(name)
 	if !ok || len(template_artifact.Reports) == 0 {
@@ -60,17 +65,22 @@ func produceReport(
 		return err
 	}
 
-	content_writer := &bytes.Buffer{}
+	parts := []*ReportPart{}
+	main := ""
 	for _, definition := range definitions {
+		content_writer := &bytes.Buffer{}
 		for _, report := range definition.Reports {
 			if report.Type != "client" {
 				continue
 			}
 
+			// Do not sanitize_html since we are writing a
+			// stand along HTML file - artifacts may
 			// generate arbitrary HTML.
 			template_engine, err := reporting.NewHTMLTemplateEngine(
 				config_obj, context.Background(), subscope,
 				vql_subsystem.NullACLManager{}, repository,
-				definition.Name)
+				definition.Name, false /* sanitize_html */)
 			if err != nil {
 				return err
 			}
@@ -87,17 +97,21 @@ func produceReport(
 
 			content_writer.Write([]byte(res))
 		}
+		parts = append(parts, &ReportPart{
+			Artifact: definition, HTML: content_writer.String()})
+		main += content_writer.String()
 	}
 
 	template_engine, err := reporting.NewHTMLTemplateEngine(
 		config_obj, context.Background(), subscope,
 		vql_subsystem.NullACLManager{}, repository,
-		template)
+		template, false /* sanitize_html */)
 	if err != nil {
 		return err
 	}
 
-	template_engine.SetEnv("main", content_writer.String())
+	template_engine.SetEnv("main", main)
+	template_engine.SetEnv("parts", parts)
 
 	result, err := template_engine.RenderRaw(
 		html_template_string, template_engine.Env.ToDict())