From 43f044fe0b39e12b3e4b88d54fb3330603ff5fcb Mon Sep 17 00:00:00 2001 From: Botong Ou Date: Thu, 16 Feb 2023 22:15:03 -0500 Subject: [PATCH 1/6] julia mnist classification example Signed-off-by: Botong Ou --- e2e/v1/docs/julia_mnist_test.go | 31 +++++++++++++++ e2e/v1/docs/testdata/julia_mnist/build.envd | 7 ++++ e2e/v1/docs/testdata/julia_mnist/mnist.jl | 43 +++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 e2e/v1/docs/julia_mnist_test.go create mode 100644 e2e/v1/docs/testdata/julia_mnist/build.envd create mode 100644 e2e/v1/docs/testdata/julia_mnist/mnist.jl diff --git a/e2e/v1/docs/julia_mnist_test.go b/e2e/v1/docs/julia_mnist_test.go new file mode 100644 index 000000000..779b6ae90 --- /dev/null +++ b/e2e/v1/docs/julia_mnist_test.go @@ -0,0 +1,31 @@ +package docs + +import ( + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + e2e "github.com/tensorchord/envd/e2e/v1" +) + +var _ = Describe("julia_mnist", Ordered, func() { + exampleName := "julia_mnist" + testcase := "e2e" + e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase) + BeforeAll(e.BuildImage(true)) + BeforeEach(e.RunContainer()) + It("test 1", func() { + res, err := e.ExecRuntimeCommand("julia-mnist") + Expect(err).To(BeNil()) + IsNumber := func(s string) bool { + _, err = strconv.ParseFloat(s, 64) + if err == nil { + return true + } else { + return false + } + } + Expect(res).To(Satisfy(IsNumber)) + }) + AfterEach(e.DestroyContainer()) +}) diff --git a/e2e/v1/docs/testdata/julia_mnist/build.envd b/e2e/v1/docs/testdata/julia_mnist/build.envd new file mode 100644 index 000000000..71db1252e --- /dev/null +++ b/e2e/v1/docs/testdata/julia_mnist/build.envd @@ -0,0 +1,7 @@ +# syntax=v1 +def build(): + base(dev=True) + + install.julia() + install.julia_packages(name=["Flux", "MLDatasets"]) + runtime.command(commands={"julia-mnist": "julia mnist.jl"}) diff --git a/e2e/v1/docs/testdata/julia_mnist/mnist.jl b/e2e/v1/docs/testdata/julia_mnist/mnist.jl new file mode 100644 index 000000000..e4327ad3a --- /dev/null +++ b/e2e/v1/docs/testdata/julia_mnist/mnist.jl @@ -0,0 +1,43 @@ +using Flux, MLDatasets +using Flux: train!, onehotbatch + +ENV["DATADEPS_ALWAYS_ACCEPT"] = "true" + +# Load training data (images, labels) +x_train, y_train = MLDatasets.MNIST(split=:train)[:] +# Load test data (images, labels) +x_test, y_test = MLDatasets.MNIST(split=:test)[:] +# Convert grayscale to float +x_train = Float32.(x_train) +# Create labels batch +y_train = Flux.onehotbatch(y_train, 0:9) + +model = Chain( + Dense(784, 256, relu), + Dense(256, 64, relu), + Dense(64, 10, relu), + softmax +) + +loss(x, y) = Flux.Losses.logitcrossentropy(model(x), y) + +optimizer = ADAM(0.0001) + +parameters = Flux.params(model) +# flatten() function converts array 28x28x60000 into 784x60000 (28*28x60000) +train_data = [(Flux.flatten(x_train), Flux.flatten(y_train))] +# Range in loop can be used smaller +for i in 1:100 + Flux.train!(loss, parameters, train_data, optimizer) +end + +test_data = [(Flux.flatten(x_test), y_test)] +accuracy = 0 +for i in 1:length(y_test) + global accuracy + if findmax(model(test_data[1][1][:, i]))[2] - 1 == y_test[i] + accuracy = accuracy + 1 + end +end + +print(accuracy / length(y_test)) \ No newline at end of file From 518cb908e6cc99dd17a72aaa2420b1d568779013 Mon Sep 17 00:00:00 2001 From: Botong Ou Date: Thu, 16 Feb 2023 22:16:28 -0500 Subject: [PATCH 2/6] julia mnist classification example Signed-off-by: Botong Ou --- e2e/v1/docs/julia_mnist_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/v1/docs/julia_mnist_test.go b/e2e/v1/docs/julia_mnist_test.go index 779b6ae90..3af856bb0 100644 --- a/e2e/v1/docs/julia_mnist_test.go +++ b/e2e/v1/docs/julia_mnist_test.go @@ -14,7 +14,7 @@ var _ = Describe("julia_mnist", Ordered, func() { e := e2e.NewExample(e2e.BuildContextDirWithName(exampleName), testcase) BeforeAll(e.BuildImage(true)) BeforeEach(e.RunContainer()) - It("test 1", func() { + It("execute runtime command `julia-mnist`", func() { res, err := e.ExecRuntimeCommand("julia-mnist") Expect(err).To(BeNil()) IsNumber := func(s string) bool { From c6ab5e927f025913a3586aa2fdd35d46bf267d9c Mon Sep 17 00:00:00 2001 From: Botong Ou Date: Thu, 16 Feb 2023 22:24:44 -0500 Subject: [PATCH 3/6] fix lint problem Signed-off-by: Botong Ou --- e2e/v1/docs/julia_mnist_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/v1/docs/julia_mnist_test.go b/e2e/v1/docs/julia_mnist_test.go index 3af856bb0..d4928ebf2 100644 --- a/e2e/v1/docs/julia_mnist_test.go +++ b/e2e/v1/docs/julia_mnist_test.go @@ -1,11 +1,11 @@ package docs import ( - "strconv" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" e2e "github.com/tensorchord/envd/e2e/v1" + + "strconv" ) var _ = Describe("julia_mnist", Ordered, func() { From f6f7a1a4f5b9a94ffcfd7f76646858bb1770e5a6 Mon Sep 17 00:00:00 2001 From: Botong Ou Date: Thu, 16 Feb 2023 22:29:22 -0500 Subject: [PATCH 4/6] fix lint problem Signed-off-by: Botong Ou --- e2e/v1/docs/julia_mnist_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/v1/docs/julia_mnist_test.go b/e2e/v1/docs/julia_mnist_test.go index d4928ebf2..74611320d 100644 --- a/e2e/v1/docs/julia_mnist_test.go +++ b/e2e/v1/docs/julia_mnist_test.go @@ -1,11 +1,12 @@ package docs import ( + "strconv" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - e2e "github.com/tensorchord/envd/e2e/v1" - "strconv" + e2e "github.com/tensorchord/envd/e2e/v1" ) var _ = Describe("julia_mnist", Ordered, func() { From 88cb07c48da9864ae1db960d9bd3f2e905df617a Mon Sep 17 00:00:00 2001 From: Botong Ou Date: Thu, 16 Feb 2023 22:32:07 -0500 Subject: [PATCH 5/6] fix another lint problem Signed-off-by: Botong Ou --- e2e/v1/docs/julia_mnist_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e/v1/docs/julia_mnist_test.go b/e2e/v1/docs/julia_mnist_test.go index 74611320d..92f035e9a 100644 --- a/e2e/v1/docs/julia_mnist_test.go +++ b/e2e/v1/docs/julia_mnist_test.go @@ -22,9 +22,8 @@ var _ = Describe("julia_mnist", Ordered, func() { _, err = strconv.ParseFloat(s, 64) if err == nil { return true - } else { - return false } + return false } Expect(res).To(Satisfy(IsNumber)) }) From 211d784f64f3f65f511c46da59987b2c585b6b71 Mon Sep 17 00:00:00 2001 From: Botong Ou Date: Thu, 16 Feb 2023 22:34:34 -0500 Subject: [PATCH 6/6] fix another lint problem Signed-off-by: Botong Ou --- e2e/v1/docs/julia_mnist_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/e2e/v1/docs/julia_mnist_test.go b/e2e/v1/docs/julia_mnist_test.go index 92f035e9a..f48292981 100644 --- a/e2e/v1/docs/julia_mnist_test.go +++ b/e2e/v1/docs/julia_mnist_test.go @@ -20,10 +20,7 @@ var _ = Describe("julia_mnist", Ordered, func() { Expect(err).To(BeNil()) IsNumber := func(s string) bool { _, err = strconv.ParseFloat(s, 64) - if err == nil { - return true - } - return false + return err == nil } Expect(res).To(Satisfy(IsNumber)) })