simagix · simagix · Jan 2, 2023 · Jan 2, 2023
diff --git a/README.md b/README.md
@@ -1,10 +1,11 @@
 # Hatchet - MongoDB JSON Log Analyzer
-Hatchet is a MongoDB JSON logs analyzer and it stores processed data in a embbedded SQLite3 database.
+Hatchet is a MongoDB JSON Log Analyzer and it stores processed data in an embedded SQLite3 database to support RESTful APIs and a web interface.  With an embedded database, Hatchet provides an interactive users experience to navigate reports and charts.
 
 ## Build
 Clone and run the *build.sh* script; *gcc* is required to support CGO.
 ```bash
-./build.sh
+git clone --depth 1 git@github.com:simagix/hatchet.git
+cd hatchet ; ./build.sh
 ```
 
 An executable *hatchet* is output to the directory *dist/*.

diff --git a/database.go b/database.go
@@ -208,15 +208,15 @@ func getTableSummary(tableName string) string {
 			module = "community"
 		}
 		if version != "" {
-			arr = append(arr, fmt.Sprintf("v%v (%v)", version, module))
+			arr = append(arr, fmt.Sprintf(": MongoDB v%v (%v)", version, module))
 		}
 		if os != "" {
 			arr = append(arr, "os: "+os)
 		}
 		if arch != "" {
 			arr = append(arr, "arch: "+arch)
 		}
-		return table + " " + strings.Join(arr, ", ")
+		return table + strings.Join(arr, ", ")
 	}
 	return ""
 }

diff --git a/html_handler.go b/html_handler.go
@@ -148,13 +148,21 @@ func htmlHandler(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Query().Get("COLLSCAN") == "true" {
 			collscan = true
 		}
-		orderBy := r.URL.Query().Get("orderBy")
-		order := "DESC"
+		var order, orderBy string
+		orderBy = r.URL.Query().Get("orderBy")
 		if orderBy == "" {
 			orderBy = "avg_ms"
 		} else if orderBy == "index" || orderBy == "_index" {
 			orderBy = "_index"
 		}
+		order = r.URL.Query().Get("order")
+		if order == "" {
+			if orderBy == "op" || orderBy == "ns" {
+				order = "ASC"
+			} else {
+				order = "DESC"
+			}
+		}
 		summary := getTableSummary(tableName)
 		ops, err := getSlowOps(tableName, orderBy, order, collscan)
 		if err != nil {

diff --git a/logv2.go b/logv2.go
@@ -28,6 +28,9 @@ var instance *Logv2
 
 // GetLogv2 returns Logv2 instance
 func GetLogv2() *Logv2 {
+	if instance == nil {
+		instance = &Logv2{dbfile: SQLITE3_FILE}
+	}
 	return instance
 }
 
@@ -38,6 +41,7 @@ type Logv2 struct {
 	filename   string
 	legacy     bool
 	tableName  string
+	testing    bool //test mode
 	totalLines int
 	verbose    bool
 }
@@ -106,17 +110,9 @@ func (ptr *Logv2) Analyze(filename string) error {
 	var reader *bufio.Reader
 	ptr.filename = filename
 	log.Println("processing", filename)
-	ptr.tableName = "hatchet"
 	dirname := filepath.Dir(ptr.dbfile)
 	os.Mkdir(dirname, 0755)
-	temp := filepath.Base(ptr.filename)
-	i := strings.LastIndex(temp, ".log")
-	if i > 0 {
-		ptr.tableName = getTableName(temp[0:i])
-	}
-	if ptr.tableName == "hatchet" {
-		ptr.tableName = "_hatchet"
-	}
+	ptr.tableName = getHatchetName(ptr.filename)
 	log.Println("hatchet table is", ptr.tableName)
 	if file, err = os.Open(filename); err != nil {
 		return err
@@ -164,7 +160,11 @@ func (ptr *Logv2) Analyze(filename string) error {
 		defer db.Close()
 
 		log.Println("creating table", ptr.tableName)
-		if _, err = db.Exec(getHatchetInitStmt(ptr.tableName)); err != nil {
+		stmts := getHatchetInitStmt(ptr.tableName)
+		if ptr.verbose {
+			log.Println(stmts)
+		}
+		if _, err = db.Exec(stmts); err != nil {
 			return err
 		}
 
@@ -182,7 +182,7 @@ func (ptr *Logv2) Analyze(filename string) error {
 	}
 
 	for {
-		if !ptr.verbose && !ptr.legacy && index%50 == 0 {
+		if !ptr.testing && !ptr.legacy && index%50 == 0 {
 			fmt.Fprintf(os.Stderr, "\r%3d%% \r", (100*index)/ptr.totalLines)
 		}
 		if buf, isPrefix, err = reader.ReadLine(); err != nil { // 0x0A separator = newline
@@ -215,7 +215,9 @@ func (ptr *Logv2) Analyze(filename string) error {
 		if ptr.legacy {
 			logstr := fmt.Sprintf("%v.000Z %-2s %-8s [%v] %v", doc.Timestamp.Format(time.RFC3339)[:19],
 				doc.Severity, doc.Component, doc.Context, doc.Message)
-			fmt.Println(logstr)
+			if !ptr.testing {
+				fmt.Println(logstr)
+			}
 			continue
 		}
 		if stat, err = AnalyzeSlowOp(&doc); err != nil {
@@ -260,12 +262,12 @@ func (ptr *Logv2) Analyze(filename string) error {
 		instr = fmt.Sprintf(`INSERT INTO hatchet (name, version, module, arch, os)
 			VALUES ('%v', '%v', '%v', '%v', '%v');`, ptr.tableName, b["version"], module, arch, os)
 	}
-	uptstr := fmt.Sprintf("DELETE FROM hatchet WHERE name = '%v';", ptr.tableName)
-	db.Exec(uptstr);
+	delstr := fmt.Sprintf("DELETE FROM hatchet WHERE name = '%v';", ptr.tableName)
+	db.Exec(delstr)
 	if _, err = db.Exec(instr); err != nil {
 		return err
 	}
-	if !ptr.verbose && !ptr.legacy {
+	if !ptr.testing && !ptr.legacy {
 		fmt.Fprintf(os.Stderr, "\r                         \r")
 	}
 	return ptr.PrintSummary()

diff --git a/logv2_test.go b/logv2_test.go
@@ -7,17 +7,17 @@ import (
 )
 
 func TestAnalyze(t *testing.T) {
-	filename := "testdata/mongod.json.log.gz"
-	logv2 := &Logv2{verbose: true}
+	filename := "testdata/mongod_ops.log.gz"
+	logv2 := &Logv2{testing: true, dbfile: SQLITE3_FILE}
 	err := logv2.Analyze(filename)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestAnalyzeLegacy(t *testing.T) {
-	filename := "testdata/mongod.json.log.gz"
-	logv2 := &Logv2{legacy: true}
+	filename := "testdata/mongod_ops.log.gz"
+	logv2 := &Logv2{testing: true, legacy: true}
 	err := logv2.Analyze(filename)
 	if err != nil {
 		t.Fatal(err)

diff --git a/template.go b/template.go
@@ -123,6 +123,7 @@ const headers = `<!DOCTYPE html>
     }
     .btn:hover {
       color: #DB4437;
+	  cursor: hand;
     }
     .button {
       background-color: #f2f2f2;
@@ -133,12 +134,6 @@ const headers = `<!DOCTYPE html>
       cursor: pointer;
       font-size: 16px;
     }
-    .head {
-      border: none;
-      outline:none;
-      color: #000;
-      font-size: 24px;
-    }
     h1 {
       font-family: "Trebuchet MS";
       font-size: 1.7em;
@@ -166,7 +161,30 @@ const headers = `<!DOCTYPE html>
 	  width: 100%;
 	  color: #000;
 	  text-align: center;
-	  }
+	}
+	select {
+	  appearance: auto;
+      font-family: "Trebuchet MS";
+	  background-color: #4285F4;
+	  color: #f2f2f2;
+	  outline: 0;
+	  border: 0;
+	  cursor: pointer;
+      border-radius: .25em;
+      font-size: 1em;
+	  padding: 5px 5px;
+	}
+	.hatchet-sel {
+      font-size: 1.25em;
+	  padding: 10px 20px;
+	}
+	.rotate45:hover {
+	  -webkit-transform: rotate(45deg);
+	  -moz-transform: rotate(45deg);
+	  -o-transform: rotate(45deg);
+	  -ms-transform: rotate(45deg);
+	  transform: rotate(45deg);
+	}
   </style>
 </head>
 
@@ -185,18 +203,14 @@ const menuHTML = `
 	}
 </script>
 <div align='center'>
-	<select id='nextChart' class='btn' style="float: right;" >
-		<button id="chart" class="btn" style="float: right;"><i class="fa fa-bar-chart"></i></button>
+	<select id='nextChart' style="float: right;" >
 		<option value=''>select a chart</option>
 		<option value='/tables/{{.Table}}/charts/slowops'>ops stats</option>
 		<option value='/tables/{{.Table}}/charts/slowops?type=counts'>ops counts</option>
 		<option value='/tables/{{.Table}}/charts/connections?type=accepted'>conns accepted</option>
 		<option value='/tables/{{.Table}}/charts/connections?type=time'>conns by time</option>
 		<option value='/tables/{{.Table}}/charts/connections?type=total'>conns by total</option>
 	</select>
-	<!--
-	<button id="chart" class="btn" style="float: right;"><i class="fa fa-bar-chart"></i></button>
-	-->
 	<button id="title"  return false;"
 		class="btn" style="float: center;"><i class="fa fa-home"></i> Hatchet</button>
 	<button id="stats"  return false;"
@@ -213,13 +227,13 @@ func getStatsTable() string {
 	<table width='100%'>
 		<tr>
 			<th>#</th>
-			<th>command</th>
-			<th>namespace</th>
-			<th>count</th>
-			<th>avg ms</th>
-			<th>max ms</th>
-			<th>total ms</th>
-			<th>total reslen</th>
+			<th>op <a href='/tables/{{.Table}}/stats/slowops?orderBy=op'><i class='fa fa-sort-asc'/></th>
+			<th>namespace <a href='/tables/{{.Table}}/stats/slowops?orderBy=ns&order=ASC'><i class='fa fa-sort-asc'/></th>
+			<th>count <a href='/tables/{{.Table}}/stats/slowops?orderBy=count'><i class='fa fa-sort-desc'/></th>
+			<th>avg ms <a href='/tables/{{.Table}}/stats/slowops?orderBy=avg_ms'><i class='fa fa-sort-desc'/></th>
+			<th>max ms <a href='/tables/{{.Table}}/stats/slowops?orderBy=max_ms'><i class='fa fa-sort-desc'/></th>
+			<th>total ms <a href='/tables/{{.Table}}/stats/slowops?orderBy=total_ms'><i class='fa fa-sort-desc'/></th>
+			<th>reslen <a href='/tables/{{.Table}}/stats/slowops?orderBy=reslen'><i class='fa fa-sort-desc'/></th>
 			<th>index</th>
 			<th>query pattern</th>
 		</tr>
@@ -310,8 +324,8 @@ func getMainPage() string {
 </script>
 
 <div align='center'>
-	<h2><img width='60' valign="middle" src='data:image/png;base64,{{ getHatchetImage }}'>Hatchet - MongoDB JSON Log Analyzer</img></h2>
-	<select id='table' class='btn' >
+	<h2><img class='rotate45' width='60' valign="middle" src='data:image/png;base64,{{ getHatchetImage }}'>Hatchet - MongoDB JSON Log Analyzer</img></h2>
+	<select id='table' class='hatchet-sel' >
 		<option value=''>select a hatchet</option>
 {{range $n, $value := .Tables}}
 		<option value='{{$value}}'>{{$value}}</option>

diff --git a/utils.go b/utils.go
@@ -4,8 +4,16 @@ package hatchet
 
 import (
 	"fmt"
+	"math/rand"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
+)
+
+const (
+	MAX_SIZE  = 128
+	TAIL_SIZE = 7
 )
 
 // ToInt converts to int
@@ -18,9 +26,26 @@ func ToInt(num interface{}) int {
 	return int(x)
 }
 
-func getTableName(name string) string {
+func replaceSpecialChars(name string) string {
 	for _, sep := range []string{"-", ".", " ", ":", ","} {
 		name = strings.ReplaceAll(name, sep, "_")
 	}
 C4FE
	return name
 }
+
+func getHatchetName(filename string) string {
+	temp := filepath.Base(filename)
+	tableName := replaceSpecialChars(temp)
+	i := strings.LastIndex(tableName, "_log")
+	if i >= 0 {
+		tableName = replaceSpecialChars(tableName[0:i])
+	}
+	if len(tableName) > MAX_SIZE {
+		tableName = tableName[:MAX_SIZE-TAIL_SIZE]
+	}
+	rand.Seed(time.Now().UnixNano())
+	b := make([]byte, TAIL_SIZE)
+	rand.Read(b)
+	tail := fmt.Sprintf("%x", b)[:TAIL_SIZE-1]
+	return fmt.Sprintf("%v_%v", tableName, tail)
+}
diff --git a/utils_test.go b/utils_test.go
@@ -0,0 +1,63 @@
+// Copyright 2022-present Kuei-chun Chen. All rights reserved.
+
+package hatchet
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestToInt(t *testing.T) {
+	input := "23"
+	value := ToInt(input)
+	if value != 23 {
+		t.Fatal("expected", 23, "but got", value)
+	}
+
+	str := ""
+	value = ToInt(str)
+	if value != 0 {
+		t.Fatal("expected", 0, "but got", value)
+	}
+}
+
+func TestReplaceSpecialChars(t *testing.T) {
+	value := "a_b_c_d_e_name"
+
+	filename := "a-b.c d:e,name"
+	fname := replaceSpecialChars(filename)
+	if value != fname {
+		t.Fatal("expected", value, "but got", fname)
+	}
+}
+
+func TestGetHatchetName(t *testing.T) {
+	filename := "mongod.log"
+	length := len(filename) - len(".log") + TAIL_SIZE
+	hatchetName := getHatchetName(filename)
+	if len(hatchetName) != length {
+		t.Fatal("expected", length, "but got", len(hatchetName))
+	}
+
+	filename = "mongod.log.gz"
+	length = len(filename) - len(".log.gz") + TAIL_SIZE
+	hatchetName = getHatchetName(filename)
+	if len(hatchetName) != length {
+		t.Fatal("expected", length, "but got", len(hatchetName))
+	}
+
+	filename = "mongod"
+	length = len(filename) + TAIL_SIZE
+	hatchetName = getHatchetName(filename)
+	if len(hatchetName) != length {
+		t.Fatal("expected", length, "but got", len(hatchetName))
+	}
+
+	filename = "filesys-shard-00-01.abcde.mongodb.net_2021-07-24T10_12_58_2021-07-25T10_12_58_mongodb.log.gz"
+	length = len(filename) + TAIL_SIZE
+	hatchetName = getHatchetName(filename)
+	modified := "filesys_shard_00_01_abcde_mongodb_net_2021_07_24T10_12_58_2021_07_25T10_12_58_mongodb"
+	if !strings.HasPrefix(hatchetName, modified) {
+		t.Fatal(modified+"_*", length, "but got", hatchetName)
+	}
+}