rkt · iaguis · Oct 9, 2017 · Sep 18, 2017 · iaguis · Oct 5, 2017
diff --git a/stage1/init/common/units.go b/stage1/init/common/units.go
@@ -328,6 +328,16 @@ func (uw *UnitWriter) SetupAppIO(p *stage1commontypes.Pod, ra *schema.RuntimeApp
 			if ok {
 				file.WriteString(fmt.Sprintf("STAGE1_LOGMODE=%s\n", logMode))
 			}
+			switch logMode {
+			case "k8s-plain":
+				kubernetesLogPath, ok := ra.Annotations.Get("coreos.com/rkt/experiment/kubernetes-log-path")
+				if !ok {
+					uw.err = fmt.Errorf("kubernetes-log-path annotation needs to be specified when k8s-plain logging mode is used")
+					return nil
+				}
+				file.WriteString(fmt.Sprintf("KUBERNETES_LOG_PATH=%s\n", kubernetesLogPath))
+			}
+
 		} else if needsTTYMux {
 			// tty mode brings in a `ttymux@.service` after-dependency (it needs to create the TTY first)
 			opts = append(opts, unit.NewUnitOption("Service", "TTYPath", filepath.Join("/rkt/iottymux", ra.Name.String(), "stage2-pts")))

diff --git a/stage1/init/init.go b/stage1/init/init.go
@@ -362,6 +362,11 @@ func getArgsEnv(p *stage1commontypes.Pod, flavor string, canMachinedRegister boo
 			args = append(args, fmt.Sprintf("--register=false"))
 		}
 
+		kubernetesLogDir, ok := p.Manifest.Annotations.Get("coreos.com/rkt/experiment/kubernetes-log-dir")
+		if ok {
+			args = append(args, fmt.Sprintf("--bind=%s:/rkt/kubernetes/log", kubernetesLogDir))
+		}
+
 		// use only dynamic libraries provided in the image
 		// from systemd v231 there's a new internal libsystemd-shared-v231.so
 		// which is present in /usr/lib/systemd

diff --git a/stage1/iottymux/iottymux.go b/stage1/iottymux/iottymux.go
@@ -410,9 +410,20 @@ func actionIOMux(statusFile string) error {
 	switch logMode {
 	case "k8s-plain":
 		var err error
-		// TODO(lucab): check what should be the target path with Euan
-		logTarget := filepath.Join(pathPrefix, appName, "logfile")
-		logFile, err = os.OpenFile(logTarget, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
+
+		// TODO(nhlfr): logPath coming from CRI/kubelet should be always a file name,
+		// but we may want to ensure that here and check that value explicitly.
+		logPath := os.Getenv("KUBERNETES_LOG_PATH")
+		logFullPath := filepath.Clean(filepath.Join("/rkt/kubernetes/log", logPath))
+
+		match, err := filepath.Match("/rkt/kubernetes/log/*", logFullPath)
+		if err != nil {
+			return fmt.Errorf("couldn't analyze the full log path %s: %s", logFullPath, err)
+		} else if !match {
+			return fmt.Errorf("log path is not inside /rkt/kubernetes/log, refusing path traversal")
+		}
+
+		logFile, err = os.OpenFile(logFullPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
 		if err != nil {
 			return err
 		}

diff --git a/tests/rkt_app_sandbox_test.go b/tests/rkt_app_sandbox_test.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
 	"strconv"
@@ -32,6 +33,13 @@ import (
 	"github.com/rkt/rkt/tests/testutils"
 )
 
+const (
+	// Number of retries to read CRI log file.
+	criLogsReadRetries = 5
+	// Delay between each retry attempt in reading CRI log file.
+	criLogsReadRetryDelay = 5 * time.Second
+)
+
 // TestAppSandboxOneApp is a basic test for `rkt app` sandbox.
 // It starts the sandbox, adds one app, starts it, and removes it.
 func TestAppSandboxAddStartRemove(t *testing.T) {
@@ -547,6 +555,73 @@ func TestAppSandboxAppAnnotations(t *testing.T) {
 	})
 }
 
+func TestAppSandboxCRILogs(t *testing.T) {
+	if TestedFlavor.Kvm || TestedFlavor.Fly {
+		t.Skip("CRI logs are not supported in kvm and fly flavors yet")
+	}
+
+	for _, tt := range []struct {
+		kubernetesLogDir  string
+		kubernetesLogPath string
+	}{
+		{
+			kubernetesLogDir:  "/tmp/rkt-test-logs",
+			kubernetesLogPath: "hello_0.log",
+		},
+	} {
+		args := []string{
+			"--annotation=coreos.com/rkt/experiment/logmode=k8s-plain",
+			fmt.Sprintf("--annotation=coreos.com/rkt/experiment/kubernetes-log-dir=%s", tt.kubernetesLogDir),
+		}
+
+		if err := os.MkdirAll(tt.kubernetesLogDir, 0777); err != nil {
+			t.Fatalf("Couldn't create directory for CRI logs: %v", err)
+		}
+		defer os.RemoveAll(tt.kubernetesLogDir)
+
+		testSandboxWithArgs(t, args, func(ctx *testutils.RktRunCtx, child *gexpect.ExpectSubprocess, podUUID string) {
+			imageName := "coreos.com/rkt-inspect/hello"
+			msg := "HelloFromAppInSandbox"
+
+			aciHello := patchTestACI("rkt-inspect-hello.aci", "--name="+imageName, "--exec=/inspect --print-msg="+msg)
+			defer os.Remove(aciHello)
+
+			combinedOutput(t, ctx.ExecCmd("fetch", "--insecure-options=image", aciHello))
+
+			combinedOutput(t, ctx.ExecCmd(
+				"app", "add", "--debug", podUUID,
+				imageName, "--name=hello", "--stdin=stream",
+				"--stdout=stream", "--stderr=stream",
+				fmt.Sprintf("--annotation=coreos.com/rkt/experiment/kubernetes-log-path=%s", tt.kubernetesLogPath),
+			))
+			combinedOutput(t, ctx.ExecCmd("app", "start", "--debug", podUUID, "--app=hello"))
+
+			// It takes some time to have iottymux unit running inside the stage1 container,
+			// so we are looking for CRI logs with a reasonable amount of retries.
+			var content []byte
+			var err error
+			for i := 0; i < criLogsReadRetries; i++ {
+				kubernetesLogFullPath := path.Join(tt.kubernetesLogDir, tt.kubernetesLogPath)
+				content, err = ioutil.ReadFile(kubernetesLogFullPath)
+				if err == nil {
+					sContent := string(content)
+					if strings.Contains(sContent, "stdout HelloFromAppInSandbox") {
+						break
+					}
+					err = fmt.Errorf("Expected CRI logs to contain 'stdout HelloFromAppInSandbox', instead got: %s", sContent)
+				} else {
+					err = fmt.Errorf("Couldn't open file with CRI logs: %v", err)
+				}
+				time.Sleep(criLogsReadRetryDelay)
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+
+		})
+	}
+}
+
 func testSandbox(t *testing.T, testFunc func(*testutils.RktRunCtx, *gexpect.ExpectSubprocess, string)) {
 	testSandboxWithArgs(t, nil, testFunc)
 }