8000 feat(hatchery): region prerequisite (#5151) · ovh/cds@05f8a8e · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 05f8a8e

Browse files
yesnaultyesnault
authored
feat(hatchery): region prerequisite (#5151)
Signed-off-by: Yvonnick Esnault <yvonnick.esnault@corp.ovh.com>
1 parent a5e7e0e commit 05f8a8e

File tree

22 files changed

+146
-80
lines changed
  • engine
  • sdk
  • tests
  • ui/src

    • 22 files changed

    +146
    -80
    lines changed

    ‎docs/content/docs/concepts/requirement/_index.md

    Lines changed: 2 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -15,6 +15,7 @@ Requirement types:
    1515
    - [Service]({{< relref "/docs/concepts/requirement/requirement_service.md" >}})
    1616
    - [Memory]({{< relref "/docs/concepts/requirement/requirement_memory.md" >}})
    1717
    - [OS & Architecture]({{< relref "/docs/concepts/requirement/requirement_os_arch.md" >}})
    18+
    - [Region]({{< relref "/docs/concepts/requirement/requirement_region.md" >}})
    1819

    1920
    A [Job]({{< relref "/docs/concepts/job.md" >}}) will be executed by a **worker**.
    2021

    @@ -26,3 +27,4 @@ You can set as many requirements as you want, following these rules:
    2627
    - Only one hostname can be set as requirement
    2728
    - Only one OS & Architecture requirement can be set at a time
    2829
    - Memory and Services requirements are available only on Docker models
    30+
    - Only one region can be set as requirement

    ‎docs/content/docs/concepts/requirement/requirement_region.md

    Lines changed: 19 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,19 @@
    1+
    ---
    2+
    title: "Region Requirement"
    3+
    weight: 4
    4+
    ---
    5+
    6+
    The `Region` prerequisite allows you to require a worker to have access to a specific region.
    7+
    8+
    A `Region` can be configured on each hatchery. With a free text as `myregion` in hatchery configuration,
    9+
    user can set a prerequisite 'region' with value `myregion` on CDS Job.
    10+
    11+
    Example of job configuration:
    12+
    ```
    13+
    jobs:
    14+
    - job: build
    15+
    requirements:
    16+
    - region: myregion
    17+
    steps:
    18+
    ...
    19+
    ```

    ‎engine/api/workermodel/registration.go

    Lines changed: 9 additions & 7 deletions
    Original file line numberDiff line numberDiff line change
    @@ -2,7 +2,6 @@ package workermodel
    22

    33
    import (
    44
    "context"
    5-
    "errors"
    65
    "strconv"
    76
    "strings"
    87
    "time"
    @@ -24,9 +23,7 @@ const (
    2423
    // setting flag need_registration to true in DB.
    2524
    func ComputeRegistrationNeeds(db gorp.SqlExecutor, allBinaryReqs sdk.RequirementList, reqs sdk.RequirementList) error {
    2625
    log.Debug("ComputeRegistrationNeeds>")
    27-
    var nbModelReq int
    28-
    var nbOSArchReq int
    29-
    var nbHostnameReq int
    26+
    var nbModelReq, nbOSArchReq, nbHostnameReq, nbRegionReq int
    3027

    3128
    for _, r := range reqs {
    3229
    switch r.Type {
    @@ -47,17 +44,22 @@ func ComputeRegistrationNeeds(db gorp.SqlExecutor, allBinaryReqs sdk.Requirement
    4744
    nbModelReq++
    4845
    case sdk.HostnameRequirement:
    4946
    nbHostnameReq++
    47+
    case sdk.RegionRequirement:
    48+
    nbRegionReq++
    5049
    }
    5150
    }
    5251

    5352
    if nbOSArchReq > 1 {
    54-
    return sdk.NewError(sdk.ErrWrongRequest, errors.New("invalid os-architecture requirement usage"))
    53+
    return sdk.NewErrorFrom(sdk.ErrWrongRequest, "invalid os-architecture requirement usage")
    5554
    }
    5655
    if nbModelReq > 1 {
    57-
    return sdk.NewError(sdk.ErrWrongRequest, errors.New("invalid model requirement usage"))
    56+
    return sdk.NewErrorFrom(sdk.ErrWrongRequest, "invalid model requirement usage")
    5857
    }
    5958
    if nbHostnameReq > 1 {
    60-
    return sdk.NewError(sdk.ErrWrongRequest, errors.New("invalid hostname requirement usage"))
    59+
    return sdk.NewErrorFrom(sdk.ErrWrongRequest, "invalid hostname requirement usage")
    60+
    }
    61+
    if nbRegionReq > 1 {
    62+
    return sdk.NewErrorFrom(sdk.ErrWrongRequest, "invalid region requirement usage")
    6163
    }
    6264

    6365
    return nil

    ‎engine/hatchery/local/local.go

    Lines changed: 6 additions & 25 deletions
    Original file line numberDiff line numberDiff line change
    @@ -178,31 +178,6 @@ func (h *HatcheryLocal) getWorkerBinaryName() string {
    178178
    return workerName
    179179
    }
    180180

    181-
    // checkCapabilities checks all requirements, foreach type binary, check if binary is on current host
    182-
    // returns an error "Exit status X" if current host misses one requirement
    183-
    func (h *HatcheryLocal) checkCapabilities(req []sdk.Requirement) ([]sdk.Requirement, error) {
    184-
    var tmp map[string]sdk.Requirement
    185-
    186-
    tmp = make(map[string]sdk.Requirement)
    187-
    for _, r := range req {
    188-
    ok, err := h.checkRequirement(r)
    189-
    if err != nil {
    190-
    return nil, err
    191-
    }
    192-
    193-
    if ok {
    194-
    tmp[r.Name] = r
    195-
    }
    196-
    }
    197-
    198-
    capa := make([]sdk.Requirement, 0, len(tmp))
    199-
    for _, r := range tmp {
    200-
    capa = append(capa, r)
    201-
    }
    202-
    203-
    return capa, nil
    204-
    }
    205-
    206181
    //Configuration returns Hatchery CommonConfiguration
    207182
    func (h *HatcheryLocal) Configuration() service.HatcheryCommonConfiguration {
    208183
    return h.Config.HatcheryCommonConfiguration
    @@ -350,6 +325,12 @@ func (h *HatcheryLocal) checkRequirement(r sdk.Requirement) (bool, error) {
    350325
    return true, nil
    351326
    case sdk.PluginRequirement:
    352327
    return true, nil
    328+
    case sdk.RegionRequirement:
    329+
    if r.Value != h.Configuration().Provision.Region {
    330+
    log.Debug("checkRequirement> job with region requirement: cannot spawn. hatchery-region:%s prerequisite:%s", h.Configuration().Provision.Region, r.Value)
    331+
    return false, nil
    332+
    }
    333+
    return true, nil
    353334
    case sdk.OSArchRequirement:
    354335
    osarch := strings.Split(r.Value, "/")
    355336
    if len(osarch) != 2 {

    ‎engine/hatchery/marathon/helper_test.go

    Lines changed: 4 additions & 2 deletions
    Original file line numberDiff line numberDiff line change
    @@ -1,10 +1,12 @@
    11
    package marathon
    22

    33
    import (
    4+
    "time"
    5+
    46
    "github.com/gambol99/go-marathon"
    5-
    "github.com/ovh/cds/sdk/cdsclient"
    67
    "gopkg.in/h2non/gock.v1"
    7-
    "time"
    8+
    9+
    "github.com/ovh/cds/sdk/cdsclient"
    810
    )
    911

    1012
    type marathonJDD struct {

    ‎engine/hatchery/marathon/marathon.go

    Lines changed: 1 addition & 2 deletions
    Original file line numberDiff line numberDiff line change
    @@ -13,14 +13,13 @@ import (
    1313
    "time"
    1414

    1515
    "github.com/dgrijalva/jwt-go"
    16-
    "github.com/ovh/cds/engine/service"
    17-
    1816
    "github.com/gambol99/go-marathon"
    1917
    "github.com/gorilla/mux"
    2018

    2119
    "github.com/ovh/cds/engine/api"
    2220
    "github.com/ovh/cds/engine/api/observability"
    2321
    "github.com/ovh/cds/engine/api/services"
    22+
    "github.com/ovh/cds/engine/service"
    2423
    "github.com/ovh/cds/sdk"
    2524
    "github.com/ovh/cds/sdk/cdsclient"
    2625
    "github.com/ovh/cds/sdk/hatchery"

    ‎engine/service/types.go

    Lines changed: 6 additions & 6 deletions
    MaxHeartbeatFailures int `toml:"maxHeartbeatFailures" default:"10" comment:"Maximum allowed consecutives failures on heatbeat routine" json:"maxHeartbeatFailures"`
    Original file line numberDiff line numberDiff line change
    @@ -40,12 +40,12 @@ type HatcheryCommonConfiguration struct {
    4040
    4141
    } `toml:"api" json:"api"`
    4242
    Provision struct {
    43-
    Disabled bool `toml:"disabled" default:"false" comment:"Disabled provisioning. Format:true or false" json:"disabled"`
    44-
    RatioService *int `toml:"ratioService" default:"50" commented:"true" comment:"Percent reserved for spawning worker with service requirement" json:"ratioService,omitempty" mapstructure:"ratioService"`
    45-
    MaxWorker int `toml:"maxWorker" default:"10" comment:"Maximum allowed simultaneous workers" json:"maxWorker"`
    46-
    MaxConcurrentProvisioning int `toml:"maxConcurrentProvisioning" default:"10" comment:"Maximum allowed simultaneous workers provisioning" json:"maxConcurrentProvisioning"`
    47-
    MaxConcurrentRegistering int `toml:"maxConcurrentRegistering" default:"2" comment:"Maximum allowed simultaneous workers registering. -1 to disable registering on this hatchery" json:"maxConcurrentRegistering"`
    48-
    RegisterFrequency int `toml:"registerFrequency" default:"60" comment:"Check if some worker model have to be registered each n Seconds" json:"registerFrequency"`
    43+
    RatioService *int `toml:"ratioService" default:"50" commented:"true" comment:"Percent reserved for spawning worker with service requirement" json:"ratioService,omitempty" mapstructure:"ratioService"`
    44+
    MaxWorker int `toml:"maxWorker" default:"10" comment:"Maximum allowed simultaneous workers" json:"maxWorker"`
    45+
    MaxConcurrentProvisioning int `toml:"maxConcurrentProvisioning" default:"10" comment:"Maximum allowed simultaneous workers provisioning" json:"maxConcurrentProvisioning"`
    46+
    MaxConcurrentRegistering int `toml:"maxConcurrentRegistering" default:"2" comment:"Maximum allowed simultaneous workers registering. -1 to disable registering on this hatchery" json:"maxConcurrentRegistering"`
    47+
    RegisterFrequency int `toml:"registerFrequency" default:"60" comment:"Check if some worker model have to be registered each n Seconds" json:"registerFrequency"`
    48+
    Region string `toml:"region" default:"" comment:"region of this hatchery - optional. With a free text as 'myregion', user can set a prerequisite 'region' with value 'myregion' on CDS Job" json:"region"`
    4949
    WorkerLogsOptions struct {
    5050
    Graylog struct {
    5151
    Host string `toml:"host" comment:"Example: thot.ovh.com" json:"host"`

    ‎engine/worker/internal/requirement.go

    Lines changed: 6 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -26,6 +26,7 @@ var requirementCheckFuncs = map[string]func(w *CurrentWorker, r sdk.Requirement)
    2626
    sdk.MemoryRequirement: checkMemoryRequirement,
    2727
    sdk.VolumeRequirement: checkVolumeRequirement,
    2828
    sdk.OSArchRequirement: checkOSArchRequirement,
    29+
    sdk.RegionRequirement: checkRegionRequirement,
    2930
    }
    3031

    3132
    func checkRequirements(ctx context.Context, w *CurrentWorker, a *sdk.Action) (bool, []sdk.Requirement) {
    @@ -209,6 +210,11 @@ func checkOSArchRequirement(w *CurrentWorker, r sdk.Requirement) (bool, error) {
    209210
    return osarch[0] == strings.ToLower(sdk.GOOS) && osarch[1] == strings.ToLower(sdk.GOARCH), nil
    210211
    }
    211212

    213+
    // region is checked by hatchery only
    214+
    func checkRegionRequirement(w *CurrentWorker, r sdk.Requirement) (bool, error) {
    215+
    return true, nil
    216+
    }
    217+
    212218
    // checkPluginDeployment returns true if current job:
    213219
    // - is not linked to a deployment integration
    214220
    // - is linked to a deployement integration, plugin well downloaded (in this func) and

    ‎sdk/exportentities/pipeline.go

    Lines changed: 7 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -60,6 +60,7 @@ type Requirement struct {
    6060
    Service ServiceRequirement `json:"service,omitempty" yaml:"service,omitempty"`
    6161
    Memory string `json:"memory,omitempty" yaml:"memory,omitempty"`
    6262
    OSArchRequirement string `json:"os-architecture,omitempty" yaml:"os-architecture,omitempty"`
    63+
    RegionRequirement string `json:"region,omitempty" yaml:"region,omitempty"`
    6364
    }
    6465

    6566
    // ServiceRequirement represents an exported sdk.Requirement of type ServiceRequirement
    @@ -151,6 +152,8 @@ func newRequirements(req []sdk.Requirement) []Requirement {
    151152
    res = append(res, Requirement{Service: ServiceRequirement{Name: r.Name, Value: r.Value}})
    152153
    case sdk.OSArchRequirement:
    153154
    res = append(res, Requirement{OSArchRequirement: r.Value})
    155+
    case sdk.RegionRequirement:
    156+
    res = append(res, Requirement{RegionRequirement: r.Value})
    154157
    case sdk.MemoryRequirement:
    155158
    res = append(res, Requirement{Memory: r.Value})
    156159
    }
    @@ -222,6 +225,10 @@ func computeJobRequirements(req []Requirement) []sdk.Requirement {
    222225
    name = r.OSArchRequirement
    223226
    val = r.OSArchRequirement
    224227
    tpe = sdk.OSArchRequirement
    228+
    } else if r.RegionRequirement != "" {
    229+
    name = "region"
    230+
    val = r.RegionRequirement
    231+
    tpe = sdk.RegionRequirement
    225232
    } else if r.Plugin != "" {
    226233
    name = r.Plugin
    227234
    val = r.Plugin

    ‎sdk/exportentities/pipeline_test.go

    Lines changed: 7 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -41,6 +41,11 @@ var (
    4141
    Type: sdk.OSArchRequirement,
    4242
    Value: "freebsd/amd64",
    4343
    },
    44+
    {
    45+
    Name: sdk.RegionRequirement,
    46+
    Type: sdk.RegionRequirement,
    47+
    Value: "graxyz",
    48+
    },
    4449
    },
    4550
    Actions: []sdk.Action{
    4651
    {
    @@ -542,6 +547,7 @@ jobs:
    542547
    requirements:
    543548
    - binary: git
    544549
    - os-archicture: freebsd/amd64
    550+
    - region: graxyz
    545551
    steps:
    546552
    - gitClone:
    547553
    branch: '{{.git.branch}}'
    @@ -568,7 +574,7 @@ jobs:
    568574
    test.NoError(t, err)
    569575

    570576
    assert.Len(t, p.Stages[0].Jobs[0].Action.Actions, 3)
    571-
    assert.Len(t, p.Stages[0].Jobs[0].Action.Requirements, 2)
    577+
    assert.Len(t, p.Stages[0].Jobs[0].Action.Requirements, 3)
    572578
    assert.Equal(t, sdk.GitCloneAction, p.Stages[0].Jobs[0].Action.Actions[0].Name)
    573579
    assert.Equal(t, sdk.ArtifactUpload, p.Stages[0].Jobs[0].Action.Actions[1].Name)
    574580
    assert.Equal(t, sdk.ServeStaticFiles, p.Stages[0].Jobs[0].Action.Actions[2].Name)

    0 commit comments

    Comments
     (0)
    0