8000 feat(api,cdsctl,sdk): add export worker model (#3774) · ovh/cds@689b82c · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 689b82c

Browse files
bnjjjrichardlt
bnjjj
authored and
richardlt
committed
feat(api,cdsctl,sdk): add export worker model (#3774)
* feat(api,cdsctl,sdk): refactor and add exportentities worker model Signed-off-by: Benjamin Coenen <benjamin.coenen@corp.ovh.com> * feat(api,cdsctl,sdk): add export worker model (#3767) Signed-off-by: Benjamin Coenen <benjamin.coenen@corp.ovh.com> * feat(api,cdsctl,sdk): add export worker model (#3767) Signed-off-by: Benjamin Coenen <benjamin.coenen@corp.ovh.com> * feat(api,cdsctl,sdk): add export worker model (#3767) Signed-off-by: Benjamin Coenen <benjamin.coenen@corp.ovh.com> * Apply suggestions from code review Co-Authored-By: bnjjj <benjamin.coenen@hotmail.com> * feat(api,cdsctl,sdk): add export worker model (#3767) Signed-off-by: Benjamin Coenen <benjamin.coenen@corp.ovh.com>
1 parent 50a3fa7 commit 689b82c

16 files changed

+601
-146
lines changed

‎cli/cdsctl/worker_model.go

Lines changed: 35 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,11 @@
11
package main
22

33
import (
4-
"bytes"
5-
"encoding/json"
64
"fmt"
75
"reflect"
86
"strings"
97

108
"github.com/spf13/cobra"
11-
yaml "gopkg.in/yaml.v2"
129

1310
"github.com/ovh/cds/cli"
1411
"github.com/ovh/cds/sdk"
@@ -26,6 +23,7 @@ func workerModel() *cobra.Command {
2623
cli.NewGetCommand(workerModelShowCmd, workerModelShowRun, nil, withAllCommandModifiers()...),
2724
cli.NewDeleteCommand(workerModelDeleteCmd, workerModelDeleteRun, nil),
2825
cli.NewCommand(workerModelImportCmd, workerModelImportRun, nil),
26+
cli.NewCommand(workerModelExportCmd, workerModelExportRun, nil, withAllCommandModifiers()...),
2927
})
3028
}
3129

@@ -101,24 +99,6 @@ For admin:
10199
},
102100
}
103101

104-
type workerModelFile struct {
105-
Name string `json:"name" yaml:"name"`
106-
Group string `json:"group" yaml:"group"`
107-
Communication string `json:"communication,omitempty" yaml:"communication,omitempty"`
108-
Provision int `json:"provision,omitempty" yaml:"provision,omitempty"`
109-
Image string `json:"image" yaml:"image"`
110-
Description string `json:"description" yaml:"description"`
111-
Type string `json:"type" yaml:"type"`
112-
Flavor string `json:"flavor,omitempty" yaml:"flavor,omitempty"`
113-
Envs map[string]string `json:"envs,omitempty" yaml:"envs,omitempty"`
114-
PatternName string `json:"pattern_name,omitempty" yaml:"pattern_name,omitempty"`
115-
Shell string `json:"shell,omitempty" yaml:"shell,omitempty"`
116-
PreCmd string `json:"pre_cmd,omitempty" yaml:"pre_cmd,omitempty"`
117-
Cmd string `json:"cmd,omitempty" yaml:"cmd,omitempty"`
118-
PostCmd string `json:"post_cmd,omitempty" yaml:"post_cmd,omitempty"`
119-
Restricted bool `json:"restricted" yaml:"restricted"`
120-
}
121-
122102
func workerModelImportRun(c cli.Values) error {
123103
force := c.GetBool("force")
124104
if c.GetString("filepath") == "" {
@@ -132,126 +112,12 @@ func workerModelImportRun(c cli.Values) error {
132112
return fmt.Errorf("Error: Cannot read file %s (%v)", filepath, err)
133113
}
134114

135-
buf := new(bytes.Buffer)
136-
if _, errR := buf.ReadFrom(reader); errR != nil {
137-
reader.Close()
138-
return fmt.Errorf("Error: cannot read file content %s : %v", filepath, errR)
139-
}
140-
reader.Close()
141-
142-
var modelInfos workerModelFile
143-
switch format {
144-
case exportentities.FormatJSON:
145-
if err := json.Unmarshal(buf.Bytes(), &modelInfos); err != nil {
146-
return fmt.Errorf("Error: cannot unmarshal json file %s : %v", filepath, err)
147-
}
148-
case exportentities.FormatYAML:
149-
if err := yaml.Unmarshal(buf.Bytes(), &modelInfos); err != nil {
150-
return fmt.Errorf("Error: cannot unmarshal yaml file %s : %v", filepath, err)
151-
}
152-
default:
153-
return fmt.Errorf("Invalid file format")
154-
}
155-
156-
var t string
157-
var modelDocker sdk.ModelDocker
158-
var modelVM sdk.ModelVirtualMachine
159-
switch modelInfos.Type {
160-
case sdk.Docker:
161-
t = sdk.Docker
162-
if modelInfos.Image == "" {
163-
sdk.Exit("Error: Docker image not provided\n")
164-
}
165-
modelDocker.Shell = modelInfos.Shell
166-
modelDocker.Image = modelInfos.Image
167-
modelDocker.Cmd = modelInfos.Cmd
168-
if modelInfos.PatternName == "" {
169-
if modelDocker.Shell == "" {
170-
sdk.Exit("Error: main shell command not provided\n")
171-
}
172-
if modelDocker.Cmd == "" {
173-
sdk.Exit("Error: main worker command not provided\n")
174-
}
175-
}
176-
177-
break
178-
case sdk.Openstack:
179-
t = sdk.Openstack
180-
d := sdk.ModelVirtualMachine{
181-
Image: modelInfos.Image,
182-
Flavor: modelInfos.Flavor,
183-
Cmd: modelInfos.Cmd,
184-
PostCmd: modelInfos.PostCmd,
185-
PreCmd: modelInfos.PreCmd,
186-
}
187-
if d.Image == "" {
188-
return fmt.Errorf("Error: Openstack image not provided")
189-
}
190-
if d.Flavor == "" {
191-
return fmt.Errorf("Error: Openstack flavor not provided")
192-
}
193-
if modelInfos.PatternName == "" {
194-
if d.Cmd == "" {
195-
return fmt.Errorf("Error: Openstack command not provided")
196-
}
197-
}
198-
modelVM = d
199-
break
200-
case sdk.VSphere:
201-
t = sdk.VSphere
202-
d := sdk.ModelVirtualMachine{
203-
Image: modelInfos.Image,
204-
Flavor: modelInfos.Flavor,
205-
Cmd: modelInfos.Cmd,
206-
PostCmd: modelInfos.PostCmd,
207-
PreCmd: modelInfos.PreCmd,
208-
}
209-
if d.Image == "" {
210-
return fmt.Errorf("Error: VSphere image not provided")
211-
}
212-
if modelInfos.PatternName == "" {
213-
if d.Cmd == "" {
214-
return fmt.Errorf("Error: VSphere main worker command empty")
215-
}
216-
}
217-
218-
modelVM = d
219-
break
220-
default:
221-
return fmt.Errorf("Unknown worker type: %s", modelInfos.Type)
222-
}
223-
224-
if modelInfos.Name == "" {
225-
return fmt.Errorf("Error: worker model name is not provided")
226-
}
227-
228-
if modelInfos.Group == "" {
229-
return fmt.Errorf("Error: group is not provided")
230-
}
231-
232-
g, err := client.GroupGet(modelInfos.Group)
115+
formatStr, _ := exportentities.GetFormatStr(format)
116+
wm, err := client.WorkerModelImport(reader, formatStr, force)
233117
if err != nil {
234-
return fmt.Errorf("Error : Unable to get group %s : %s", modelInfos.Group, err)
235-
}
236-
237-
if force {
238-
if existingWm, err := client.WorkerModel(modelInfos.Name); err != nil {
239-
if _, errAdd := client.WorkerModelAdd(modelInfos.Name, t, modelInfos.PatternName, &modelDocker, &modelVM, g.ID); errAdd != nil {
240-
return fmt.Errorf("Error: cannot add worker model %s (%s)", modelInfos.Name, errAdd)
241-
}
242-
fmt.Printf("Worker model %s added with success", modelInfos.Name)
243-
} else {
244-
if _, errU := client.WorkerModelUpdate(existingWm.ID, modelInfos.Name, t, &modelDocker, &modelVM, g.ID); errU != nil {
245-
return fmt.Errorf("Error: cannot update worker model %s (%s)", modelInfos.Name, errU)
246-
}
247-
fmt.Printf("Worker model %s updated with success", modelInfos.Name)
248-
}
249-
} else {
250-
if _, errAdd := client.WorkerModelAdd(modelInfos.Name, t, modelInfos.PatternName, &modelDocker, &modelVM, g.ID); errAdd != nil {
251-
return fmt.Errorf("Error: cannot add worker model %s (%s)", modelInfos.Name, errAdd)
252-
}
253-
fmt.Printf("Worker model %s added with success", modelInfos.Name)
118+
return err
254119
}
120+
fmt.Printf("Worker model %s imported with success\n", wm.Name)
255121
}
256122

257123
return nil
@@ -291,3 +157,33 @@ func workerModelDeleteRun(v cli.Values) error {
291157
}
292158
return nil
293159
}
160+
161+
var workerModelExportCmd = cli.Command{
162+
Name: "export",
163+
Short: "Export a worker model",
164+
Args: []cli.Arg{
165+
{Name: "name"},
166+
},
167+
Flags: []cli.Flag{
168+
{
169+
Kind: reflect.String,
170+
Name: "format",
171+
Usage: "Specify export format (json or yaml)",
172+
Default: "yaml",
173+
},
174+
},
175+
}
176+
177+
func workerModelExportRun(c cli.Values) error {
178+
wmName := c.GetString("name")
179+
wm, err := client.WorkerModel(wmName)
180+
if err != nil {
181+
return sdk.WrapError(err, "cannot load worker model %s", wmName)
182+
}
183+
btes, err := client.WorkerModelExport(wm.ID, c.GetString("format"))
184+
if err != nil {
185+
return err
186+
}
187+
fmt.Println(string(btes))
188+
return nil
189+
}

‎engine/api/api_routes.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,7 @@ func (api *API) InitRouter() {
416416

417417
// Worker models
418418
r.Handle("/worker/model", r.POST(api.addWorkerModelHandler), r.GET(api.getWorkerModelsHandler))
419+
r.Handle("/worker/model/import", r.POST(api.postWorkerModelImportHandler))
419420
r.Handle("/worker/model/pattern", r.POST(api.postAddWorkerModelPatternHandler, NeedAdmin(true)), r.GET(api.getWorkerModelPatternsHandler))
420421
r.Handle("/worker/model/pattern/{type}/{name}", r.GET(api.getWorkerModelPatternHandler), r.PUT(api.putWorkerModelPatternHandler, NeedAdmin(true)), r.DELETE(api.deleteWorkerModelPatternHandler, NeedAdmin(true)))
421422
r.Handle("/worker/model/book/{permModelID}", r.PUT(api.bookWorkerModelHandler, NeedHatchery()))
@@ -424,6 +425,7 @@ func (api *API) InitRouter() {
424425
r.Handle("/worker/model/type", r.GET(api.getWorkerModelTypesHandler))
425426
r.Handle("/worker/model/communication", r.GET(api.getWorkerModelCommunicationsHandler))
426427
r.Handle("/worker/model/{permModelID}", r.PUT(api.updateWorkerModelHandler), r.DELETE(api.deleteWorkerModelHandler))
428+
r.Handle("/worker/model/{permModelID}/export", r.GET(api.getWorkerModelExportHandler))
427429
r.Handle("/worker/model/{permModelID}/usage", r.GET(api.getWorkerModelUsageHandler))
428430
r.Handle("/worker/model/capability/type", r.GET(api.getRequirementTypesHandler))
429431

‎engine/api/group/group.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,22 @@ func LoadGroupByID(db gorp.SqlExecutor, id int64) (*sdk.Group, error) {
105105
}, nil
106106
}
107107

108+
// LoadGroupByName retrieves group informations from database given his name
109+
func LoadGroupByName(db gorp.SqlExecutor, name string) (*sdk.Group, error) {
110+
query := `SELECT "group".id FROM "group" WHERE "group".name = $1`
111+
var id int64
112+
if err := db.QueryRow(query, name).Scan(&id); err != nil {
113+
if err == sql.ErrNoRows {
114+
err = sdk.ErrGroupNotFound
115+
}
116+
return nil, sdk.WithStack(err)
117+
}
118+
return &sdk.Group{
119+
ID: id,
120+
Name: name,
121+
}, nil
122+
}
123+
108124
// LoadUserGroup retrieves all group users from database
109125
func LoadUserGroup(db gorp.SqlExecutor, group *sdk.Group) error {
110126
query := `SELECT "user".username, "user".data, "group_user".group_admin FROM "user"

‎engine/api/worker/model_export.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package worker
2+
3+
import (
4+
"io"
5+
6+
"github.com/ovh/cds/sdk"
7+
"github.com/ovh/cds/sdk/exportentities"
8+
)
9+
10+
// Export convert sdk.Model to an exportentities.WorkerModel, format and write into a io.Writer
11+
func Export(wm sdk.Model, f exportentities.Format, w io.Writer) (int, error) {
12+
eWm := exportentities.NewWorkerModel(wm)
13+
14+
// Marshal to the desired format
15+
b, err := exportentities.Marshal(eWm, f)
16+
if err != nil {
17+
return 0, sdk.WithStack(err)
18+
}
19+
20+
return w.Write(b)
21+
}

‎engine/api/worker/model_import.go

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package worker
2+
3+
import (
4+
"database/sql"
5+
6+
"github.com/go-gorp/gorp"
7+
"github.com/lib/pq"
8+
9+
"github.com/ovh/cds/engine/api/database/gorpmapping"
10+
"github.com/ovh/cds/engine/api/group"
11+
"github.com/ovh/cds/sdk"
12+
"github.com/ovh/cds/sdk/exportentities"
13+
)
14+
15+
// ParseAndImport parse and import an exportentities.WorkerModel
16+
func ParseAndImport(db gorp.SqlExecutor, eWorkerModel *exportentities.WorkerModel, force bool, u *sdk.User) (*sdk.Model, error) {
17+
sdkWm, err := eWorkerModel.GetWorkerModel()
18+
if err != nil {
19+
return nil, err
20+
}
21+
22+
gr, err := group.LoadGroupByName(db, sdkWm.Group.Name)
23+
if err != nil {
24+
return nil, sdk.WrapError(err, "Unable to get group %s", sdkWm.Group.Name)
25+
}
26+
sdkWm.Group = *gr
27+
sdkWm.GroupID = gr.ID
28+
29+
var modelPattern *sdk.ModelPattern
30+
if sdkWm.PatternName != "" {
31+
var errP error
32+
modelPattern, errP = LoadWorkerModelPatternByName(db, sdkWm.Type, sdkWm.PatternName)
33+
if errP != nil {
34+
return nil, sdk.WrapError(sdk.ErrWrongRequest, "Cannot load worker model pattern %s : %v", sdkWm.PatternName, errP)
35+
}
36+
}
37+
38+
//User must be admin of the group set in the model
39+
var isGroupAdmin bool
40+
currentUGroup:
41+
for _, g := range u.Groups {
42+
if g.ID == sdkWm.GroupID {
43+
for _, a := range g.Admins {
44+
if a.ID == u.ID {
45+
isGroupAdmin = true
46+
break currentUGroup
47+
}
48+
}
49+
}
50+
}
51+
52+
//User should have the right permission or be admin
53+
if !u.Admin && !isGroupAdmin {
54+
return nil, sdk.ErrWorkerModelNoAdmin
55+
}
56+
57+
switch sdkWm.Type {
58+
case sdk.Docker:
59+
if sdkWm.ModelDocker.Image == "" {
60+
return nil, sdk.WrapError(sdk.ErrWrongRequest, "Invalid worker image")
61+
}
62+
if !u.Admin && !sdkWm.Restricted {
63+
if modelPattern == nil {
64+
return nil, sdk.ErrWorkerModelNoPattern
65+
}
66+
}
67+
if sdkWm.ModelDocker.Cmd == "" || sdkWm.ModelDocker.Shell == "" {
68+
return nil, sdk.WrapError(sdk.ErrWrongRequest, "Invalid worker command or invalid shell command")
69+
}
70+
default:
71+
if sdkWm.ModelVirtualMachine.Image == "" {
72+
return nil, sdk.WrapError(sdk.ErrWrongRequest, "Invalid worker command or invalid image")
73+
}
74+
if !u.Admin && !sdkWm.Restricted {
75+
if modelPattern == nil {
76+
return nil, sdk.ErrWorkerModelNoPattern
77+
}
78+
sdkWm.ModelVirtualMachine.PreCmd = modelPattern.Model.PreCmd
79+
sdkWm.ModelVirtualMachine.Cmd = modelPattern.Model.Cmd
80+
sdkWm.ModelVirtualMachine.PostCmd = modelPattern.Model.PostCmd
81+
}
82+
}
83+
84+
if sdkWm.GroupID == 0 {
85+
return nil, sdk.WrapError(sdk.ErrWrongRequest, "groupID should be set")
86+
}
87+
88+
if group.IsDefaultGroupID(sdkWm.GroupID) {
89+
return nil, sdk.WrapError(sdk.ErrWrongRequest, "this group can't be owner of a worker model")
90+
}
91+
92+
// provision is allowed only for CDS Admin
93+
// or by currentUser with a restricted model
94+
if !u.Admin && !sdkWm.Restricted {
95+
sdkWm.Provision = 0
96+
}
97+
98+
if force {
99+
if existingWm, err := LoadWorkerModelByName(db, sdkWm.Name); err != nil {
100+
if sdk.Cause(err) == sql.ErrNoRows {
101+
if errAdd := InsertWorkerModel(db, &sdkWm); errAdd != nil {
102+
return nil, sdk.WrapError(errAdd, "cannot add worker model %s", sdkWm.Name)
103+
}
104+
} else {
105+
return nil, sdk.WrapError(err, "cannot find worker model %s", sdkWm.Name)
106+
}
107+
} else {
108+
sdkWm.ID = existingWm.ID
109+
if errU := UpdateWorkerModel(db, &sdkWm); errU != nil {
110+
return nil, sdk.WrapError(errU, "cannot update worker model %s", sdkWm.Name)
111+
}
112+
}
113+
return &sdkWm, nil
114+
}
115+
116+
if errAdd := InsertWorkerModel(db, &sdkWm); errAdd != nil {
117+
if errPG, ok := sdk.Cause(errAdd).(*pq.Error); ok && errPG.Code == gorpmapping.ViolateUniqueKeyPGCode {
118+
errAdd = sdk.ErrConflict
119+
}
120+
return nil, sdk.WrapError(errAdd, "cannot add worker model %s", sdkWm.Name)
121+
}
122+
123+
return &sdkWm, nil
124+
}

0 commit comments

Comments
 (0)
0