8000 feat(api,cdsctl,sdk): add migrations handling (#3746) · ovh/cds@3b5d805 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content
8000

Commit 3b5d805

Browse files
bnjjjbnjjj
authored
feat(api,cdsctl,sdk): add migrations handling (#3746)
close #3739
1 parent f895cfd commit 3b5d805

29 files changed

+728
-90
lines changed

‎cli/cdsctl/admin.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ func adminCommands() []*cobra.Command {
1818
adminHooks(),
1919
adminPlatformModels(),
2020
adminMaintenance(),
21+
adminMigrations(),
2122
adminPlugins(),
2223
adminBroadcasts(),
2324
adminErrors(),

‎cli/cdsctl/admin_migrations.go

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/ovh/cds/sdk"
7+
"github.com/spf13/cobra"
8+
9+
"github.com/ovh/cds/cli"
10+
)
11+
12+
var adminMigrationsCmd = cli.Command{
13+
Name: "migration",
14+
Short: "Manage CDS Migrations",
15+
}
16+
17+
func adminMigrations() *cobra.Command {
18+
return cli.NewCommand(adminMigrationsCmd, nil, []*cobra.Command{
19+
cli.NewListCommand(adminMigrationsList, adminMigrationsListFunc, nil),
20+
cli.NewCommand(adminMigrationsCancel, adminMigrationsCancelFunc, nil),
21+
cli.NewCommand(adminMigrationsReset, adminMigrationsResetFunc, nil),
22+
})
23+
}
24+
25+
var adminMigrationsList = cli.Command{
26+
Name: "list",
27+
Short: "List all CDS migrations and their states",
28+
}
29+
30+
func adminMigrationsListFunc(_ cli.Values) (cli.ListResult, error) {
31+
migrations, err := client.AdminCDSMigrationList()
32+
if err != nil {
33+
return nil, err
34+
}
35+
return cli.AsListResult(migrations), nil
36+
}
37+
38+
var adminMigrationsCancel = cli.Command{
39+
Name: "cancel",
40+
Short: "Cancel a CDS migration (USE WITH CAUTION)",
41+
Args: []cli.Arg{
42+
{Name: "id"},
43+
},
44+
}
45+
46+
func adminMigrationsCancelFunc(v cli.Values) error {
47+
id, err := v.GetInt64("id")
48+
if err != nil {
49+
return sdk.WrapError(err, "Bad id format")
50+
}
51+
52+
if err := client.AdminCDSMigrationCancel(id); err != nil {
53+
return err
54+
}
55+
fmt.Printf("Migration %d is canceled\n", id)
56+
return nil
57+
}
58+
59+
var adminMigrationsReset = cli.Command{
60+
Name: "reset",
61+
Short: `Reset a CDS migration, so basically it put the migration status to "TO DO" (USE WITH CAUTION)`,
62+
Args: []cli.Arg{
63+
{Name: "id"},
64+
},
65+
}
66+
67+
func adminMigrationsResetFunc(v cli.Values) error {
68+
id, err := v.GetInt64("id")
69+
if err != nil {
70+
return sdk.WrapError(err, "Bad id format")
71+
}
72+
73+
if err := client.AdminCDSMigrationReset(id); err != nil {
74+
return err
75+
}
76+
fmt.Printf("Migration %d is reset\n", id)
77+
return nil
78+
}

‎cli/cdsctl/application_key.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package main
22

33
import (
4+
"fmt"
5+
46
"github.com/spf13/cobra"
57

68
"github.com/ovh/cds/cli"
@@ -40,7 +42,13 @@ func applicationCreateKeyRun(v cli.Values) error {
4042
Type: v["key-type"],
4143
},
4244
}
43-
return client.ApplicationKeyCreate(v[_ProjectKey], v[_ApplicationName], key)
45+
if err := client.ApplicationKeyCreate(v[_ProjectKey], v[_ApplicationName], key); err != nil {
46+
return err
47+
}
48+
49+
fmt.Printf("Application key %s of type %s created with success in application %s\n", key.Name, key.Type, v[_ApplicationName])
50+
fmt.Println(key.Public)
51+
return nil
4452
}
4553

4654
var applicationKeyListCmd = cli.Command{

‎cli/cdsctl/environment_key.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package main
22

33
import (
4+
"fmt"
5+
46
"github.com/spf13/cobra"
57

68
"github.com/ovh/cds/cli"
@@ -40,7 +42,13 @@ func environmentCreateKeyRun(v cli.Values) error {
4042
Type: v["key-type"],
4143
},
4244
}
43-
return client.EnvironmentKeyCreate(v[_ProjectKey], v["env-name"], key)
45+
if err := client.EnvironmentKeyCreate(v[_ProjectKey], v["env-name"], key); err != nil {
46+
return err
47+
}
48+
49+
fmt.Printf("Environment key %s of type %s created with success in environment %s\n", key.Name, key.Type, v["env-name"])
50+
fmt.Println(key.Public)
51+
return nil
4452
}
4553

4654
var environmentKeyListCmd = cli.Command{

‎cli/cdsctl/project_key.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package main
22

33
import (
4+
"fmt"
5+
46
"github.com/spf13/cobra"
57

68
"github.com/ovh/cds/cli"
@@ -39,7 +41,13 @@ func projectCreateKeyRun(v cli.Values) error {
47D8 3941
Type: v["key-type"],
4042
},
4143
}
42-
return client.ProjectKeyCreate(v[_ProjectKey], key)
44+
if err := client.ProjectKeyCreate(v[_ProjectKey], key); err != nil {
45+
return err
46+
}
47+
48+
fmt.Printf("Project key %s of type %s created with success in project %s\n", key.Name, key.Type, v[_ProjectKey])
49+
fmt.Println(key.Public)
50+
return nil
4351
}
4452

4553
var projectKeyListCmd = cli.Command{

‎docs/content/hosting/ready-to-run/from-binaries.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ This article contains the steps to start CDS locally, with API, UI and a local H
1515
### Prerequisite
1616

1717
- a Redis
18-
- a PostgreSQL 9.4 min
18+
- a PostgreSQL 9.5 min
1919

2020
### Get latest release from GitHub
2121

‎docs/content/hosting/requirements.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ There is are two ways to set up CDS:
1515

1616
## CDS API Third-parties
1717

18-
At the minimum, CDS needs a PostgreSQL database >= 9.4 and Redis >= 3.2. But for serious usage your may need:
18+
At the minimum, CDS needs a PostgreSQL database >= 9.5 and Redis >= 3.2. But for serious usage your may need:
1919

2020
- A [Redis](https://redis.io) server or sentinels based cluster used as a cache and session store
2121
- A LDAP Server for authentication

‎engine/api/action/action.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,7 @@ func GetPipelineUsingAction(db gorp.SqlExecutor, name string) ([]PipelineUsingAc
441441
LEFT OUTER JOIN workflow_node ON workflow_node.pipeline_id = pipeline.id
442442
LEFT OUTER JOIN workflow ON workflow_node.workflow_id = workflow.id
443443
LEFT JOIN action as actionChild ON actionChild.id = child_id
444-
WHERE actionChild.name = $1 and actionChild.public = true
444+
WHERE actionChild.name = $1 and actionChild.public = true AND pipeline.name IS NOT NULL
445445
ORDER BY projectkey, appName, pipName, actionName;
446446
`
447447
rows, errq := db.Query(query, name)

‎engine/api/admin.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import (
1818
func (api *API) postMaintenanceHandler() service.Handler {
1919
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
2020
enable := FormString(r, "enable")
21-
api.Cache.Publish(maintenanceQueueName, enable)
21+
api.Cache.Publish(sdk.MaintenanceQueueName, enable)
2222
return nil
2323
}
2424
}

‎engine/api/api.go

Lines changed: 52 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import (
66
"io"
77
"net/http"
88
"os"
9+
"strings"
910
"time"
1011

12+
"github.com/blang/semver"
1113
"github.com/go-gorp/gorp"
1214
"github.com/gorilla/mux"
1315
"go.opencensus.io/stats"
@@ -36,6 +38,7 @@ import (
3638
"github.com/ovh/cds/engine/api/secret"
3739
"github.com/ovh/cds/engine/api/services"
3840
"github.com/ovh/cds/engine/api/sessionstore"
41+
"github.com/ovh/cds/engine/api/version"
3942
"github.com/ovh/cds/engine/api/warning"
4043
"github.com/ovh/cds/engine/api/worker"
4144
"github.com/ovh/cds/engine/api/workflow"
@@ -684,30 +687,61 @@ func (a *API) Serve(ctx context.Context) error {
684687
sdk.GoRoutine(ctx, "services.KillDeadServices", func(ctx context.Context) {
685688
services.KillDeadServices(ctx, a.mustDB)
686689
}, a.PanicDump())
687-
sdk.GoRoutine(ctx, "migrate.CleanOldWorkflow", func(ctx context.Context) {
688-
migrate.CleanOldWorkflow(ctx, a.Cache, a.DBConnectionFactory.GetDBMap, a.Config.URL.API)
689-
}, a.PanicDump())
690-
sdk.GoRoutine(ctx, "migrate.KeyMigration", func(ctx context.Context) {
691-
migrate.KeyMigration(a.Cache, a.DBConnectionFactory.GetDBMap, &sdk.User{Admin: true})
692-
}, a.PanicDump())
693690
sdk.GoRoutine(ctx, "broadcast.Initialize", func(ctx context.Context) {
694691
broadcast.Initialize(ctx, a.DBConnectionFactory.GetDBMap)
695692
}, a.PanicDump())
696693
sdk.GoRoutine(ctx, "api.serviceAPIHeartbeat", func(ctx context.Context) {
697694
a.serviceAPIHeartbeat(ctx)
698695
}, a.PanicDump())
699-
sdk.GoRoutine(ctx, "migrate.WorkflowData", func(ctx context.Context) {
700-
migrate.MigrateToWorkflowData(a.DBConnectionFactory.GetDBMap, a.Cache)
701-
}, a.PanicDump())
702696

703697
//Temporary migration code
704-
go migrate.WorkflowNodeRunArtifacts(a.Cache, a.DBConnectionFactory.GetDBMap)
698+
//DEPRECATED Migrations
699+
sdk.GoRoutine(ctx, "migrate.KeyMigration", func(ctx context.Context) {
700+
migrate.CleanOldWorkflow(ctx, a.Cache, a.DBConnectionFactory.GetDBMap, a.Config.URL.API)
701+
}, a.PanicDump())
702+
703+
//DEPRECATED Migrations
704+
sdk.GoRoutine(ctx, "migrate.KeyMigration", func(ctx context.Context) {
705+
migrate.KeyMigration(a.Cache, a.DBConnectionFactory.GetDBMap, &sdk.User{Admin: true})
706+
}, a.PanicDump())
707+
708+
migrate.Add(sdk.Migration{Name: "WorkflowData", Release: "0.37.0", Mandatory: true, ExecFunc: func(ctx context.Context) error {
709+
return migrate.MigrateToWorkflowData(a.DBConnectionFactory.GetDBMap, a.Cache)
710+
}})
705711
if os.Getenv("CDS_MIGRATE_ENABLE") == "true" {
706-
go func() {
707-
if err := migrate.MigrateActionDEPRECATEDGitClone(a.mustDB, a.Cache); err != nil {
708-
log.Error("Bootstrap Error: %v", err)
712+
migrate.Add(sdk.Migration{Name: "MigrateActionDEPRECATEDGitClone", Release: "0.37.0", Mandatory: true, ExecFunc: func(ctx context.Context) error {
713+
return migrate.MigrateActionDEPRECATEDGitClone(a.mustDB, a.Cache)
714+
}})
715+
}
716+
// migrate.Add(sdk.Migration{Name: "GitClonePrivateKey", Release: "0.37.0", Mandatory: true, ExecFunc: func(ctx context.Context) error {
717+
// return migrate.GitClonePrivateKey(a.mustDB, a.Cache)
718+
// }})
719+
720+
isFreshInstall, errF := version.IsFreshInstall(a.mustDB())
721+
if errF != nil {
722+
return sdk.WrapError(errF, "Unable to check if it's a fresh installation of CDS")
723+
}
724+
725+
if isFreshInstall {
726+
if err := migrate.SaveAllMigrations(a.mustDB()); err != nil {
727+
return sdk.WrapError(err, "Cannot save all migrations to done")
728+
}
729+
} else {
730+
if sdk.VersionCurrent().Version != "" && !strings.HasPrefix(sdk.VersionCurrent().Version, "snapshot") {
731+
major, minor, _, errV := version.MaxVersion(a.mustDB())
732+
if errV != nil {
733+
return sdk.WrapError(errV, "Cannot fetch max version of CDS already started")
709734
}
710-
}()
735+
if major != 0 || minor != 0 {
736+
minSemverCompatible, _ := semver.Parse(migrate.MinCompatibleRelease)
737+
if major < minSemverCompatible.Major || (major == minSemverCompatible.Major && minor < minSemverCompatible.Minor) {
738+
return fmt.Errorf("there are some mandatory migrations which aren't done. Please check each changelog of CDS. Maybe you have skipped a release migration. The minimum compatible release is %s, please update to this release before", migrate.MinCompatibleRelease)
739+
}
740+
}
741+
}
742+
743+
// Run all migrations in several goroutines
744+
migrate.Run(ctx, a.mustDB(), a.PanicDump())
711745
}
712746

713747
// Init Services
@@ -790,6 +824,10 @@ func (a *API) Serve(ctx context.Context) error {
790824
}
791825
}()
792826

827+
if err := version.Upsert(a.mustDB()); err != nil {
828+
return sdk.WrapError(err, "Cannot upsert cds version")
829+
}
830+
793831
log.Info("Starting CDS API HTTP Server on %s:%d", a.Config.HTTP.Addr, a.Config.HTTP.Port)
794832
if err := s.ListenAndServe(); err != nil {
795833
return fmt.Errorf("Cannot start HTTP server: %v", err)

0 commit comments

Comments
 (0)
0