8000 Treat action code as attachments by chetanmeh · Pull Request #3945 · apache/openwhisk · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Treat action code as attachments #3945 8000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Aug 20, 2018
Merged

Treat action code as attachments #3945

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
91c795a
Treat action code as attachments for all newly created or update acti…
dubee Oct 9, 2017
0dd9f66
Fix issues post rebase
chetanmeh Aug 2, 2018
58d0f92
Store base64 encoded streams in raw form
chetanmeh Aug 2, 2018
6ddada3
Fix testcases
chetanmeh Aug 6, 2018
79684ef
Add test for exec deserialization compatibility
chetanmeh Aug 7, 2018
3b3baa0
Support code stored in jar property
chetanmeh Aug 7, 2018
fe7dac2
Use octet stream type for binary and text/plain otherwise
chetanmeh Aug 7, 2018
152fa79
Reword exception message to state that code can be string on object
chetanmeh Aug 7, 2018
1bb3986
Drop rewrite of jar files as base64 encoded file
chetanmeh Aug 7, 2018
a649de4
Remove unnecessary special handling of java code
chetanmeh Aug 7, 2018
a25b0f3
Fix expected attachmentType
chetanmeh Aug 7, 2018
3e3eb3a
Remove unused variable
chetanmeh Aug 7, 2018
745c9f6
Add some more compat tests
chetanmeh Aug 7, 2018
7ab5666
Simplify json deserialization based on Markus patch
chetanmeh Aug 7, 2018
dd3d942
Use std charset
chetanmeh Aug 7, 2018
b142bc1
Remove dead code
chetanmeh Aug 7, 2018
ac5baab
Add support for tweaking code size via CODE_SIZE
chetanmeh Aug 14, 2018
0982e1e
Use attachment for php runtime also
chetanmeh Aug 15, 2018 8000
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 63 additions & 11 deletions ansible/files/runtimes.json
View file
Open in desktop
6D40
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@
"name": "nodejsaction",
"tag": "latest"
},
"deprecated": true
"deprecated": true,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
},
{
"kind": "nodejs:6",
Expand All @@ -19,6 +23,10 @@
"tag": "latest"
},
"deprecated": false,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
},
"stemCells": [{
"count": 2,
"memory": "256 MB"
Expand All @@ -32,7 +40,11 @@
"name": "action-nodejs-v8",
"tag": "latest"
},
"deprecated": false
"deprecated": false,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
}
],
"python": [
Expand All @@ -43,7 +55,11 @@
"name": "python2action",
"tag": "latest"
},
"deprecated": false
"deprecated": false,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
},
{
"kind": "python:2",
Expand All @@ -53,7 +69,11 @@
"name": "python2action",
"tag": "latest"
},
"deprecated": false
"deprecated": false,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
},
{
"kind": "python:3",
Expand All @@ -62,7 +82,11 @@
"name": "python3action",
"tag": "latest"
},
"deprecated": false
"deprecated": false,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
}
],
"swift": [
Expand All @@ -73,7 +97,11 @@
"name": "swiftaction",
"tag": "latest"
},
"deprecated": true
"deprecated": true,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
},
{
"kind": "swift:3",
Expand All @@ -82,7 +110,11 @@
"name": "swift3action",
"tag": "latest"
},
"deprecated": true
"deprecated": true,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
},
{
"kind": "swift:3.1.1",
Expand All @@ -91,7 +123,11 @@
"name": "action-swift-v3.1.1",
"tag": "latest"
},
"deprecated": false
"deprecated": false,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
},
{
"kind": "swift:4.1",
Expand All @@ -101,7 +137,11 @@
"name": "action-swift-v4.1",
"tag": "latest"
},
"deprecated": false
"deprecated": false,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
}
],
"java": [
Expand All @@ -115,8 +155,8 @@
},
"deprecated": false,
"attached": {
"attachmentName": "jarfile",
"attachmentType": "application/java-archive"
"attachmentName": "codefile",
"attachmentType": "text/plain"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This won’t break pre-existing java actions I presume.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes attachmentName and attachmentType are not used in actual flow. ContentType is just stored but not interpreted so far for any purpose

},
"requireMain": true
}
Expand All @@ -130,6 +170,10 @@
"prefix": "openwhisk",
"name": "action-php-v7.1",
"tag": "latest"
},
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
},
{
Expand All @@ -140,6 +184,10 @@
"prefix": "openwhisk",
"name": "action-php-v7.2",
"tag": "latest"
},
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
}
}
],
Expand All @@ -148,6 +196,10 @@
"kind": "ruby:2.5",
"default": true,
"deprecated": false,
"attached": {
"attachmentName": "codefile",
"attachmentType": "text/plain"
},
"image": {
"prefix": "openwhisk",
"name": "action-ruby-v2.5",
Expand Down
24 changes: 13 additions & 11 deletions common/scala/src/main/scala/whisk/core/entity/Exec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,14 +140,14 @@ protected[core] case class CodeExecMetaDataAsString(manifest: RuntimeManifest,

protected[core] case class CodeExecAsAttachment(manifest: RuntimeManifest,
override val code: Attachment[String],
override val entryPoint: Option[String])
override val entryPoint: Option[String],
override val binary: Boolean = false)
extends CodeExec[Attachment[String]] {
override val kind = manifest.kind
override val image = manifest.image
override val sentinelledLogs = manifest.sentinelledLogs.getOrElse(true)
override val deprecated = manifest.deprecated.getOrElse(false)
override val pull = false
override lazy val binary = true
override def codeAsJson = code.toJson

def inline(bytes: Array[Byte]): CodeExecAsAttachment = {
Expand Down Expand Up @@ -301,22 +301,24 @@ protected[core] object Exec extends ArgNormalizer[Exec] with DefaultJsonProtocol
}

manifest.attached
.map { a =>
val jar: Attachment[String] = {
// java actions once stored the attachment in "jar" instead of "code"
obj.fields.get("code").orElse(obj.fields.get("jar"))
} map {
attFmt[String].read(_)
} getOrElse {
.map { _ =>
// java actions once stored the attachment in "jar" instead of "code"
val code = obj.fields.get("code").orElse(obj.fields.get("jar")).getOrElse {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we can remove this code or jar (separately). It’s been a long time since I made this schema change.

throw new DeserializationException(
s"'code' must be a valid base64 string in 'exec' for '$kind' actions")
s"'code' must be a string or attachment object defined in 'exec' for '$kind' actions")
}
val binary: Boolean = code match {
case JsString(c) => isBinaryCode(c)
case _ => obj.fields.get("binary").map(_.convertTo[Boolean]).getOrElse(false)
}

val main = optMainField.orElse {
if (manifest.requireMain.exists(identity)) {
throw new DeserializationException(s"'main' must be a string defined in 'exec' for '$kind' actions")
} else None
}
CodeExecAsAttachment(manifest, jar, main)

CodeExecAsAttachment(manifest, attFmt[String].read(code), main, binary)
}
.getOrElse {
val code: String = obj.fields.get("code") match {
Expand Down
29 changes: 18 additions & 11 deletions common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@ package whisk.core.entity

import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.nio.charset.StandardCharsets
import java.util.Base64

import akka.http.scaladsl.model.ContentTypes

import scala.concurrent.ExecutionContext
import scala.concurrent.Future
import scala.util.{Failure, Success, Try}
Expand Down Expand Up @@ -333,23 +336,27 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[
require(doc != null, "doc undefined")
} map { _ =>
doc.exec match {
case exec @ CodeExecAsAttachment(_, Inline(code), _) =>
case exec @ CodeExecAsAttachment(_, Inline(code), _, binary) =>
implicit val logger = db.logging
implicit val ec = db.executionContext

val stream = new ByteArrayInputStream(Base64.getDecoder().decode(code))
val manifest = exec.manifest.attached.get
val (bytes, attachmentType) = if (binary) {
(Base64.getDecoder.decode(code), ContentTypes.`application/octet-stream`)
} else {
(code.getBytes(StandardCharsets.UTF_8), ContentTypes.`text/plain(UTF-8)`)
}
val stream = new ByteArrayInputStream(bytes)
val oldAttachment = old
.flatMap(_.exec match {
case CodeExecAsAttachment(_, a: Attached, _) => Some(a)
case _ => None
case CodeExecAsAttachment(_, a: Attached, _, _) => Some(a)
case _ => None
})

super.putAndAttach(
db,
doc,
(d, a) => d.copy(exec = exec.attach(a)).revision[WhiskAction](d.rev),
manifest.attachmentType,
attachmentType,
stream,
oldAttachment,
Some { a: WhiskAction =>
Expand Down Expand Up @@ -378,12 +385,12 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[

fa.flatMap { action =>
action.exec match {
case exec @ CodeExecAsAttachment(_, attached: Attached, _) =>
case exec @ CodeExecAsAttachment(_, attached: Attached, _, binary) =>
val boas = new ByteArrayOutputStream()
val b64s = Base64.getEncoder().wrap(boas)
val wrapped = if (binary) Base64.getEncoder().wrap(boas) else boas

getAttachment[A](db, action, attached, b64s, Some { a: WhiskAction =>
b64s.close()
getAttachment[A](db, action, attached, wrapped, Some { a: WhiskAction =>
wrapped.close()
val newAction = a.copy(exec = exec.inline(boas.toByteArray))
newAction.revision(a.rev)
newAction
Expand All @@ -397,7 +404,7 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[

def attachmentHandler(action: WhiskAction, attached: Attached): WhiskAction = {
val eu = action.exec match {
case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _, _, _), _) =>
case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _, _, _), _, _) =>
require(
attachmentName == attached.attachmentName,
s"Attachment name '${attached.attachmentName}' does not match the expected name '$attachmentName'")
Expand Down
11 changes: 9 additions & 2 deletions tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ class ColdBlockingInvokeSimulation extends Simulation {
val host = sys.env("OPENWHISK_HOST")

val users: Int = sys.env("USERS").toInt
val codeSize: Int = sys.env.getOrElse("CODE_SIZE", "0").toInt
val seconds: FiniteDuration = sys.env.getOrElse("SECONDS", "10").toInt.seconds
val actionsPerUser: Int = sys.env.getOrElse("ACTIONS_PER_USER", "5").toInt

Expand Down Expand Up @@ -64,8 +65,7 @@ class ColdBlockingInvokeSimulation extends Simulation {
openWhisk("Create action")
.authenticate(uuid, key)
.action(actionName)
.create(FileUtils
.readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8)))
.create(actionCode))
}.rendezVous(users)
// Execute all actions for the given amount of time.
.during(seconds) {
Expand All @@ -81,6 +81,13 @@ class ColdBlockingInvokeSimulation extends Simulation {
}
}

private def actionCode = {
val code = FileUtils
.readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8)
//Pad the code with empty space to increase the stored code size
if (codeSize > 0) code + " " * codeSize else code
}

setUp(test.inject(atOnceUsers(users)))
.protocols(openWhiskProtocol)
// One failure will make the build yellow
Expand Down
Loading
0