8000 feat(api,ui,worker): add serve static files (close #3567) (#3618) · ovh/cds@b12ddfa · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit b12ddfa

Browse files
bnjjjsguiheux
bnjjj
authored and
sguiheux
committed
feat(api,ui,worker): add serve static files (close #3567) (#3618)
1 parent 5c7edc8 commit b12ddfa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1131
-33
lines changed

‎docs/content/workflows/pipelines/actions/builtin/serve-static-files.md

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
+++
2+
title = "Serve Static Files"
3+
chapter = true
4+
+++
5+
6+
**Serve Static Files Action** is a builtin action, you can't modify it.
7+
8+
This action can be used to upload static files and serve them. For example your HTML report about coverage, tests, performances, ...
9+
10+
Pay attention this action is only available if your objectstore is configured to use Openstack Swift. And fow now by default your static files will be deleted after 2 months.
11+
12+
## Parameters
13+
* name: Name to display in CDS UI and identify your static files
14+
* path: Path where static files will be uploaded (example: mywebsite/*). If it's a file, the entrypoint would be set to this filename by default.
15+
* tag: Filename (and not path) for the entrypoint when serving static files (default: if empty it would be index.html).
16+
17+
### Example
18+
19+
* In this example I created a website with script in bash and use action `Serve Static Files`.
20+
21+
![img](/images/workflows.pipelines.actions.builtin.serve-static-files-job.png)
22+
23+
* Launch pipeline, check logs
24+
25+
![img](/images/workflows.pipelines.actions.builtin.serve-static-files-logs.png)
26+
27+
* View your static files with links in the tab artifact
28+
29+
![img](/images/workflows.pipelines.actions.builtin.serve-static-files-tab.png)

‎docs/static/images/workflows.pipelines.actions.builtin.serve-static-files-job.png

78.1 KB
Loading

‎docs/static/images/workflows.pipelines.actions.builtin.serve-static-files-logs.png

35.6 KB
Loading

‎docs/static/images/workflows.pipelines.actions.builtin.serve-static-files-tab.png

28.8 KB
Loading

‎engine/api/action/builtin.go

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,32 @@ Semver used if fully compatible with https://semver.org/
248248
Description: "Set a list of artifacts, separate by , . You can also use regexp.",
249249
Type: sdk.StringParameter,
250250
})
251-
return checkBuiltinAction(db, gitrelease)
251+
if err := checkBuiltinAction(db, gitrelease); err != nil {
252+
return err
253+
}
254+
255+
// ----------------------------------- Serve Static Files -----------------------
10000 256+
serveStaticAct := sdk.NewAction(sdk.ServeStaticFiles)
257+
serveStaticAct.Type = sdk.BuiltinAction
258+
serveStaticAct.Description = `CDS Builtin Action
259+
Useful to upload static files and serve them.
260+
For example your report about coverage, tests, performances, ...`
261+
serveStaticAct.Parameter(sdk.Parameter{
262+
Name: "name",
263+
Description: "Name to display in CDS UI and identify your static files",
264+
Type: sdk.StringParameter})
265+
serveStaticAct.Parameter(sdk.Parameter{
266+
Name: "path",
267+
Description: "Path where static files will be uploaded (example: mywebsite/*). If it's a file, the entrypoint would be set to this filename by default.",
268+
Type: sdk.StringParameter})
269+
serveStaticAct.Parameter(sdk.Parameter{
270+
Name: "entrypoint",
271+
Description: "Filename (and not path) for the entrypoint when serving static files (default: if empty it would be index.html)",
272+
Type: sdk.StringParameter,
273+
Value: "",
274+
Advanced: true})
275+
276+
return checkBuiltinAction(db, serveStaticAct)
252277
}
253278

254279
// checkBuiltinAction add builtin actions in database if needed

‎engine/api/api_routes.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,7 @@ func (api *API) InitRouter() {
303303
r.Handle("/project/{key}/application/{permApplicationName}/pipeline/{permPipelineKey}/{buildNumber}/artifact/{tag}/url/callback", r.POSTEXECUTE(api.postArtifactWithTempURLCallbackHandler))
304304
r.Handle("/project/{key}/application/{permApplicationName}/pipeline/{permPipelineKey}/{buildNumber}/artifact/{tag}", r.POSTEXECUTE(api.uploadArtifactHandler))
305305
r.Handle("/project/{key}/application/{permApplicationName}/pipeline/{permPipelineKey}/artifact/download/{id}", r.GET(api.downloadArtifactHandler))
306+
r.Handle("/staticfiles/store", r.GET(api.getStaticFilesStoreHandler, Auth(false)))
306307
r.Handle("/artifact/store", r.GET(api.getArtifactsStoreHandler, Auth(false)))
307308
r.Handle("/artifact/{hash}", r.GET(api.downloadArtifactDirectHandler, Auth(false)))
308309

@@ -345,6 +346,9 @@ func (api *API) InitRouter() {
345346
r.Handle("/queue/workflows/{permID}/artifact/{ref}", r.POSTEXECUTE(api.postWorkflowJobArtifactHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
346347
r.Handle("/queue/workflows/{permID}/artifact/{ref}/url", r.POSTEXECUTE(api.postWorkflowJobArtifacWithTempURLHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
347348
r.Handle("/queue/workflows/{permID}/artifact/{ref}/url/callback", r.POSTEXECUTE(api.postWorkflowJobArtifactWithTempURLCallbackHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
349+
r.Handle("/queue/workflows/{permID}/staticfiles/{name}", r.POSTEXECUTE(api.postWorkflowJobStaticFilesHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
350+
r.Handle("/queue/workflows/{permID}/staticfiles/{name}/url", r.POSTEXECUTE(api.postWorkflowJobStaticFilesWithTempURLHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
351+
r.Handle("/queue/workflows/{permID}/staticfiles/{name}/url/callback", r.POSTEXECUTE(api.postWorkflowJobStaticFilesWithTempURLCallbackHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
348352

349353
r.Handle("/variable/type", r.GET(api.getVariableTypeHandler))
350354
r.Handle("/parameter/type", r.GET(api.getParameterTypeHandler))

‎engine/api/objectstore/fs.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ func (fss *FilesystemStore) Status() sdk.MonitoringStatusLine {
3434
return sdk.MonitoringStatusLine{Component: "Object-Store", Value: "Filesystem Storage", Status: sdk.MonitoringStatusOK}
3535
}
3636

37+
// ServeStaticFiles NOT YET IMPLEMENTED
38+
func (fss *FilesystemStore) ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error) {
39+
return "", sdk.ErrNotImplemented
40+
}
41+
3742
// Store store a object on disk
3843
func (fss *FilesystemStore) Store(o Object, data io.ReadCloser) (string, error) {
3944
dst := path.Join(fss.basedir, o.GetPath())

‎engine/api/objectstore/objectstore.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@ func Store(o Object, data io.ReadCloser) (string, error) {
3939
return "", fmt.Errorf("store not initialized")
4040
}
4141

42+
//ServeStaticFiles send files to serve static files with the entrypoint of html page and return public URL taking a tar file
43+
func ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error) {
44+
if storage != nil {
45+
return storage.ServeStaticFiles(o, entrypoint, data)
46+
}
47+
return "", fmt.Errorf("store not initialized")
48+
}
49+
4250
//Fetch an object with default objectstore driver
4351
func Fetch(o Object) (io.ReadCloser, error) {
4452
if storage != nil {
@@ -76,6 +84,7 @@ func FetchTempURL(o Object) (string, error) {
7684
type Driver interface {
7785
Status() sdk.MonitoringStatusLine
7886
Store(o Object, data io.ReadCloser) (string, error)
87+
ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error)
7988
Fetch(o Object) (io.ReadCloser, error)
8089
Delete(o Object) error
8190
}
@@ -86,6 +95,10 @@ type DriverWithRedirect interface {
8695
StoreURL(o Object) (url string, key string, err error)
8796
// FetchURL returns a temporary url and a secret key to fetch an object
8897
FetchURL(o Object) (url string, key string, err error)
98+
// ServeStaticFilesURL returns a temporary url and a secret key to serve static files in a container
99+
ServeStaticFilesURL(o Object, entrypoint string) (string, string, error)
100+
// GetPublicURL returns a public url to fetch an object (check your object ACLs before)
101+
GetPublicURL(o Object) (url string, err error)
89102
}
90103

91104
// Initialize setup wanted ObjectStore driver

‎engine/api/objectstore/ssh.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"io/ioutil"
99

1010
"github.com/ovh/cds/sdk"
11+
1112
"golang.org/x/crypto/ssh"
1213
)
1314

‎engine/api/objectstore/swift.go

Lines changed: 70 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,13 +49,48 @@ func (s *SwiftStore) Status() sdk.MonitoringStatusLine {
4949
}
5050
}
5151

52+
// ServeStaticFiles send files to serve static files with the entrypoint of html page and return public URL taking a tar file
53+
func (s *SwiftStore) ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error) {
54+
container := s.containerprefix + o.GetPath()
55+
object := o.GetName()
56+
escape(container, object)
57+
log.Debug("SwiftStore> Storing /%s/%s\n", container, object)
58+
59+
if entrypoint == "" {
60+
entrypoint = "index.html"
61+
}
62+
headers := map[string]string{
63+
"X-Web-Mode": "TRUE",
64+
"X-Container-Meta-Web-Listings": "TRUE",
65+
"X-Container-Meta-Web-Index": entrypoint,
66+
"X-Container-Read": ".r:*,.rlistings",
67+
"X-Delete-After": fmt.Sprintf("%d", time.Now().Add(time.Hour*1500).Unix()), //TODO: to delete when purge will be developed
68+
}
69+
70+
log.Debug("SwiftStore> creating container %s", container)
71+
if err := s.ContainerCreate(container, headers); err != nil {
72+
return "", sdk.WrapError(err, "Unable to create container %s", container)
73+
}
74+
75+
log.Debug("SwiftStore> creating object %s/%s", container, object)
76+
res, errU := s.BulkUpload(container, data, "tar", nil)
77+
if errU != nil {
78+
return "", sdk.WrapError(errU, "SwiftStore> Unable to bulk upload %s : %v : %+v", object, errU, res.Errors)
79+
}
80+
81+
if err := data.Close(); err != nil {
82+
return "", sdk.WrapError(err, "Unable to close data buffer")
83+
}
84+
85+
return s.StorageUrl + "/" + container, nil
86+
}
87+
5288
// Store stores in swift
5389
func (s *SwiftStore) Store(o Object, data io.ReadCloser) (string, error) {
5490
container := s.containerprefix + o.GetPath()
5591
object := o.GetName()
5692
escape(container, object)
5793
log.Debug("SwiftStore> Storing /%s/%s\n", container, object)
58-
5994
log.Debug("SwiftStore> creating container %s", container)
6095
if err := s.ContainerCreate(container, nil); err != nil {
6196
return "", sdk.WrapError(err, "Unable to create container %s", container)
@@ -129,7 +164,6 @@ func (s *SwiftStore) StoreURL(o Object) (string, string, error) {
129164
container := s.containerprefix + o.GetPath()
130165
object := o.GetName()
131166
escape(container, object)
132-
133167
if err := s.ContainerCreate(container, nil); err != nil {
134168
return "", "", sdk.WrapError(err, "Unable to create container %s", container)
135169
}
@@ -143,6 +177,39 @@ func (s *SwiftStore) StoreURL(o Object) (string, string, error) {
143177
return url, string(key), nil
144178
}
145179

180+
// GetPublicURL returns a public url to fetch an object (check your object ACLs before)
181+
func (s *SwiftStore) GetPublicURL(o Object) (url string, err error) {
182+
return s.StorageUrl + "/" + (s.containerprefix + o.GetPath()), nil
183+
}
184+
185+
// ServeStaticFilesURL returns a temporary url and a secret key to serve static files in a container
186+
func (s *SwiftStore) ServeStaticFilesURL(o Object, entrypoint string) (string, string, error) {
187+
container := s.containerprefix + o.GetPath()
188+
object := o.GetName()
189+
escape(container, object)
190+
if entrypoint == "" {
191+
entrypoint = "index.html"
192+
}
193+
194+
headers := map[string]string{
195+
"X-Web-Mode": "TRUE",
196+
"X-Container-Meta-Web-Listings": "TRUE",
197+
"X-Container-Meta-Web-Index": entrypoint,
198+
"X-Container-Read": ".r:*,.rlistings",
199+
}
200+
if err := s.ContainerCreate(container, headers); err != nil {
201+
return "", "", sdk.WrapError(err, "Unable to create container %s", container)
202+
}
203+
204+
key, err := s.containerKey(container)
205+
if err != nil {
206+
return "", "", sdk.WrapError(err, "Unable to get container key %s", container)
207+
}
208+
209+
url := s.ObjectTempUrl(container, object, string(key), "PUT", time.Now().Add(time.Hour))
210+
return url, string(key), nil
211+
}
212+
146213
func (s *SwiftStore) containerKey(container string) (string, error) {
147214
_, headers, err := s.Container(container)
148215
if err != nil {
@@ -178,5 +245,5 @@ func (s *SwiftStore) FetchURL(o Object) (string, string, error) {
178245
url := s.ObjectTempUrl(container, object, string(key), "GET", time.Now().Add(time.Hour))
179246

180247
log.Debug("SwiftStore> Fetch URL: %s", string(url))
181-
return url, string(key), nil
248+
return url + "&extract-archive=tar.gz", string(key), nil
182249
}

0 commit comments

Comments
 (0)
0