8000 feat(engine): add aws s3 object store (#4195) · ovh/cds@5faec5f · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 5faec5f

Browse files
Nic Pattersonyesnault
Nic Patterson
authored and
yesnault
committed
feat(engine): add aws s3 object store (#4195)
1 parent cedba97 commit 5faec5f

File tree

10 files changed

+291
-66
lines changed

10 files changed

+291
-66
lines changed

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
*.swp
22
*.iml
33
.idea
4-
*.iml
4+
vendor/
55
.vscode
66
.DS_Store
77
dump.rdb

‎engine/api/api.go

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ type Configuration struct {
106106
From string `toml:"from" default:"no-reply@cds.local" json:"from"`
107107
} `toml:"smtp" comment:"#####################\n# CDS SMTP Settings \n####################" json:"smtp"`
108108
Artifact struct {
109-
Mode string `toml:"mode" default:"local" comment:"swift or local" json:"mode"`
109+
Mode string `toml:"mode" default:"local" comment:"swift, awss3 or local" json:"mode"`
110110
Local struct {
111111
BaseDirectory string `toml:"baseDirectory" default:"/tmp/cds/artifacts" json:"baseDirectory"`
112112
} `toml:"local"`
@@ -120,6 +120,17 @@ type Configuration struct {
120120
ContainerPrefix string `toml:"containerPrefix" comment:"Use if your want to prefix containers for CDS Artifacts" json:"containerPrefix"`
121121
DisableTempURL bool `toml:"disableTempURL" default:"false" commented:"true" comment:"True if you want to disable Temporary URL in file upload" json:"disableTempURL"`
122122
} `toml:"openstack" json:"openstack"`
123+
AWSS3 struct {
124+
BucketName string `toml:"bucketName" json:"bucketName" comment:"Name of the S3 bucket to use when storing artifacts"`
125+
Region string `toml:"region" json:"region" default:"us-east-1" comment:"The AWS region"`
126+
Prefix string `toml:"prefix" json:"prefix" comment:"A subfolder of the bucket to store objects in, if left empty will store at the root of the bucket"`
127+
AuthFromEnvironment bool `toml:"authFromEnv" json:"authFromEnv" default:"false" comment:"Pull S3 auth information from env vars AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID"`
128+
SharedCredsFile string `toml:"sharedCredsFile" json:"sharedCredsFile" comment:"The path for the AWS credential file, used with profile"`
129+
Profile string `toml:"profile" json:"profile" comment:"The profile within the AWS credentials file to use"`
130+
AccessKeyID string `toml:"accessKeyId" json:"accessKeyId" comment:"A static AWS Secret Key ID"`
131+
SecretAccessKey string `toml:"secretAccessKey" json:"secretAccessKey" comment:"A static AWS Secret Access Key"`
132+
SessionToken string `toml:"sessionToken" json:"sessionToken" comment:"A static AWS session token"`
133+
} `toml:"awss3" json:"awss3"`
123134
} `toml:"artifact" comment:"Either filesystem local storage or Openstack Swift Storage are supported" json:"artifact"`
124135
Events struct {
125136
Kafka struct {
@@ -326,7 +337,7 @@ func (a *API) CheckConfiguration(config interface{}) error {
326337
}
327338

328339
switch aConfig.Artifact.Mode {
329-
case "local", "openstack", "swift":
340+
case "local", "awss3", "openstack", "swift":
330341
default:
331342
return fmt.Errorf("Invalid artifact mode")
332343
}
@@ -549,6 +560,8 @@ func (a *API) Serve(ctx context.Context) error {
549560
objectstoreKind = objectstore.Openstack
550561
case "swift":
551562
objectstoreKind = objectstore.Swift
563+
case "awss3":
564+
objectstoreKind = objectstore.AWSS3
552565
case "filesystem", "local":
553566
objectstoreKind = objectstore.Filesystem
554567
default:
@@ -558,6 +571,17 @@ func (a *API) Serve(ctx context.Context) error {
558571
cfg := objectstore.Config{
559572
Kind: objectstoreKind,
560573
Options: objectstore.ConfigOptions{
574+
AWSS3: objectstore.ConfigOptionsAWSS3{
575+
Prefix: a.Config.Artifact.AWSS3.Prefix,
576+
SecretAccessKey: a.Config.Artifact.AWSS3.SecretAccessKey,
577+
AccessKeyID: a.Config.Artifact.AWSS3.AccessKeyID,
578+
Profile: a.Config.Artifact.AWSS3.Profile,
579+
SharedCredsFile: a.Config.Artifact.AWSS3.SharedCredsFile,
580+
AuthFromEnvironment: a.Config.Artifact.AWSS3.AuthFromEnvironment,
581+
BucketName: a.Config.Artifact.AWSS3.BucketName,
582+
Region: a.Config.Artifact.AWSS3.Region,
583+
SessionToken: a.Config.Artifact.AWSS3.SessionToken,
584+
},
561585
Openstack: objectstore.ConfigOptionsOpenstack{
562586
Address: a.Config.Artifact.Openstack.URL,
563587
Username: a.Config.Artifact.Openstack.Username,

‎engine/api/objectstore/awss3.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package objectstore
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"io"
7+
"io/ioutil"
8+
"path"
9+
10+
"github.com/aws/aws-sdk-go/aws"
11+
"github.com/aws/aws-sdk-go/aws/credentials"
12+
"github.com/aws/aws-sdk-go/aws/session"
13+
"github.com/aws/aws-sdk-go/service/s3"
14+
"github.com/aws/aws-sdk-go/service/s3/s3manager"
15+
16+
"github.com/ovh/cds/sdk"
17+
"github.com/ovh/cds/sdk/log"
18+
)
19+
20+
// FilesystemStore implements ObjectStore interface with filesystem driver
21+
type AWSS3Store struct {
22+
projectIntegration sdk.ProjectIntegration
23+
prefix string
24+
bucketName string
25+
region string
26+
sess *session.Session
27+
}
28+
29+
func newS3Store(integration sdk.ProjectIntegration, conf ConfigOptionsAWSS3) (*AWSS3Store, error) {
30+
log.Info("ObjectStore> Initialize AWS S3 driver for bucket: %s in region %s", conf.BucketName, conf.Region)
31+
aConf := aws.NewConfig()
32+
aConf.Region = aws.String(conf.Region)
33+
if conf.AuthFromEnvironment {
34+
aConf.Credentials = credentials.NewEnvCredentials()
35+
} else if conf.Profile != "" {
36+
// if the shared creds file is empty the AWS SDK will check the defaults automatically
37+
aConf.Credentials = credentials.NewSharedCredentials(conf.SharedCredsFile, conf.Profile)
38+
} else {
39+
aConf.Credentials = credentials.NewStaticCredentials(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken)
40+
}
41+
sess, err := session.NewSession(aConf)
42+
if err != nil {
43+
return nil, sdk.WrapError(err, "Unable to create an AWS session")
44+
}
45+
return &AWSS3Store{
46+
bucketName: conf.BucketName,
47+
prefix: conf.Prefix,
48+
projectIntegration: integration,
49+
sess: sess,
50+
}, nil
51+
}
52+
53+
func (s *AWSS3Store) account() (*s3.ListObjectsOutput, error) {
54+
log.Debug("AWS-S3-Store> Getting bucket info\n")
55+
s3n := s3.New(s.sess)
56+
out, err := s3n.ListObjects(&s3.ListObjectsInput{
57+
Bucket: aws.String(s.bucketName),
58+
Prefix: aws.String(s.prefix),
59+
})
60+
if err != nil {
61+
return nil, sdk.WrapError(err, "AWS-S3-Store> Unable to read data from input object")
62+
}
63+
64+
return out, nil
65+
}
66+
67+
func (s *AWSS3Store) getObjectPath(o Object) string {
68+
return path.Join(s.prefix, o.GetPath(), o.GetName())
69+
}
70+
71+
func (s *AWSS3Store) TemporaryURLSupported() bool {
72+
return false
73+
}
74+
75+
func (s *AWSS3Store) GetProjectIntegration() sdk.ProjectIntegration {
76+
return s.projectIntegration
77+
}
78+
79+
func (s *AWSS3Store) Status() sdk.MonitoringStatusLine {
80+
out, err := s.account()
81+
if err != nil {
82+
return sdk.MonitoringStatusLine{Component: "Object-Store", Value: "AWSS3 KO" + err.Error(), Status: sdk.MonitoringStatusAlert}
83+
}
84+
return sdk.MonitoringStatusLine{
85+
Component: "Object-Store",
86+
Value: fmt.Sprintf("S3 OK (%d objects)", len(out.Contents)),
87+
Status: sdk.MonitoringStatusOK,
88+
}
89+
}
90+
91+
func (s *AWSS3Store) Store(o Object, data io.ReadCloser) (string, error) {
92+
defer data.Close()
93+
log.Debug("AWS-S3-Store> Setting up uploader\n")
94+
uploader := s3manager.NewUploader(s.sess)
95+
b, e := ioutil.ReadAll(data)
96+
if e != nil {
97+
return "", sdk.WrapError(e, "AWS-S3-Store> Unable to read data from input object")
98+
}
99+
100+
log.Debug("AWS-S3-Store> Uploading object %s to bucket %s\n", s.getObjectPath(o), s.bucketName)
101+
out, err := uploader.Upload(&s3manager.UploadInput{
102+
Key: aws.String(s.getObjectPath(o)),
103+
Bucket: aws.String(s.bucketName),
104+
Body: bytes.NewReader(b),
105+
})
106+
if err != nil {
107+
return "", sdk.WrapError(err, "AWS-S3-Store> Unable to create object %s", s.getObjectPath(o))
108+
}
109+
log.Debug("AWS-S3-Store> Successfully uploaded object %s to bucket %s\n", s.getObjectPath(o), s.bucketName)
110+
return out.Location, nil
111+
}
112+
113+
func (s *AWSS3Store) ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error) {
114+
return "", sdk.ErrNotImplemented
115+
}
116+
117+
func (s *AWSS3Store) Fetch(o Object) (io.ReadCloser, error) {
118+
s3n := s3.New(s.sess)
119+
log.Debug("AWS-S3-Store> Fetching object %s from bucket %s\n", s.getObjectPath(o), s.bucketName)
120+
out, err := s3n.GetObject(&s3.GetObjectInput{
121+
Key: aws.String(s.getObjectPath(o)),
122+
Bucket: aws.String(s.bucketName),
123+
})
124+
if err != nil {
125+
return nil, sdk.WrapError(err, "AWS-S3-Store> Unable to download object %s", s.getObjectPath(o))
126+
}
127+
log.Debug("AWS-S3-Store> Successfully retrieved object %s from bucket %s\n", s.getObjectPath(o), s.bucketName)
128+
return out.Body, nil
129+
}
130+
131+
func (s *AWSS3Store) Delete(o Object) error {
132+
s3n := s3.New(s.sess)
133+
log.Debug("AWS-S3-Store> Deleting object %s from bucket %s\n", s.getObjectPath(o), s.bucketName)
134+
_, err := s3n.DeleteObject(&s3.DeleteObjectInput{
135+
Key: aws.String(s.getObjectPath(o)),
136+
Bucket: aws.String(s.bucketName),
137+
})
138+
if err != nil {
139+
return sdk.WrapError(err, "AWS-S3-Store> Unable to delete object %s", s.getObjectPath(o))
140+
}
141+
log.Debug("AWS-S3-Store> Successfully Deleted object %s/%s\n", s.bucketName, s.getObjectPath(o))
142+
return nil
143+
}

‎engine/api/objectstore/fs.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@ type FilesystemStore struct {
1717
}
1818

1919
// newFilesystemStore creates a new ObjectStore with filesystem driver
20-
func newFilesystemStore(projectIntegration sdk.ProjectIntegration, basedir string) (*FilesystemStore, error) {
21-
log.Info("Objectstore> Initialize Filesystem driver on directory: %s", basedir)
22-
if basedir == "" {
20+
func newFilesystemStore(projectIntegration sdk.ProjectIntegration, conf ConfigOptionsFilesystem) (*FilesystemStore, error) {
21+
log.Info("ObjectStore> Initialize Filesystem driver on directory: %s", conf.Basedir)
22+
if conf.Basedir == "" {
2323
return nil, fmt.Errorf("artifact storage is filesystem, but --artifact-basedir is not provided")
2424
}
2525

26-
fss := &FilesystemStore{projectIntegration: projectIntegration, basedir: basedir}
26+
fss := &FilesystemStore{projectIntegration: projectIntegration, basedir: conf.Basedir}
2727
return fss, nil
2828
}
2929

‎engine/api/objectstore/objectstore.go

Lines changed: 35 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"strings"
99

1010
"github.com/go-gorp/gorp"
11-
"github.com/ncw/swift"
1211

1312
"github.com/ovh/cds/engine/api/integration"
1413
"github.com/ovh/cds/sdk"
@@ -45,11 +44,12 @@ type Kind int
4544
// These are the defined objecstore drivers
4645
const (
4746
Openstack Kind = iota
47+
AWSS3
4848
Filesystem
4949
Swift
5050
)
5151

52-
// Config represents all the configuration for all objecstore drivers
52+
// Config represents all the configuration for all objectstore drivers
5353
type Config struct {
5454
IntegrationName string
5555
ProjectName string
@@ -59,10 +59,25 @@ type Config struct {
5959

6060
// ConfigOptions is used by Config
6161
type ConfigOptions struct {
62+
AWSS3 ConfigOptionsAWSS3
6263
Openstack ConfigOptionsOpenstack
6364
Filesystem ConfigOptionsFilesystem
6465
}
6566

67+
// ConfigOptionsAWSS3 is used by ConfigOptions
68+
type ConfigOptionsAWSS3 struct {
69+
Region string
70+
BucketName string
71+
Prefix string
72+
// Auth options, can provide a profile name, from environment or directly provide access keys
73+
AuthFromEnvironment bool
74+
SharedCredsFile string
75+
Profile string
76+
AccessKeyID string
77+
SecretAccessKey string
78+
SessionToken string
79+
}
80+
6681
// ConfigOptionsOpenstack is used by ConfigOptions
6782
type ConfigOptionsOpenstack struct {
6883
Address string
@@ -91,53 +106,32 @@ func InitDriver(db gorp.SqlExecutor, projectKey, integrationName string) (Driver
91106
return nil, fmt.Errorf("projectIntegration.Model %t is not a storage integration", projectIntegration.Model.Storage)
92107
}
93108

94-
if projectIntegration.Model.Name == sdk.OpenstackIntegrationModel {
95-
s := SwiftStore{
96-
Connection: swift.Connection{
97-
AuthUrl: projectIntegration.Config["address"].Value,
98-
Region: projectIntegration.Config["region"].Value,
99-
Tenant: projectIntegration.Config["tenant_name"].Value,
100-
Domain: projectIntegration.Config["domain"].Value,
101-
UserName: projectIntegration.Config["username"].Value,
102-
ApiKey: projectIntegration.Config["password"].Value,
103-
},
104-
containerprefix: projectIntegration.Config["storage_container_prefix"].Value,
105-
disableTempURL: projectIntegration.Config["storage_temporary_url_supported"].Value == "false",
106-
projectIntegration: projectIntegration,
107-
}
108-
109-
if err := s.Authenticate(); err != nil {
110-
return nil, sdk.WrapError(err, "Unable to authenticate")
111-
}
112-
return &s, nil
109+
switch projectIntegration.Model.Name {
110+
case sdk.OpenstackIntegrationModel:
111+
return newSwiftStore(projectIntegration, ConfigOptionsOpenstack{
112+
Address: projectIntegration.Config["address"].Value,
113+
Region: projectIntegration.Config["region"].Value,
114+
Tenant: projectIntegration.Config["tenant_name"].Value,
115+
Domain: projectIntegration.Config["domain"].Value,
116+
Username: projectIntegration.Config["username"].Value,
117+
Password: projectIntegration.Config["password"].Value,
118+
ContainerPrefix: projectIntegration.Config["storage_container_prefix"].Value,
119+
DisableTempURL: projectIntegration.Config["storage_temporary_url_supported"].Value == "false",
120+
})
121+
default:
122+
return nil, fmt.Errorf("Invalid Integration %s", projectIntegration.Model.Name)
113123
}
114-
115-
return nil, fmt.Errorf("Invalid Integration %s", projectIntegration.Model.Name)
116124
}
117125

118126
// Init initialise a new ArtifactStorage
119127
func Init(c context.Context, cfg Config) (Driver, error) {
120128
switch cfg.Kind {
121129
case Openstack, Swift:
122-
s := SwiftStore{
123-
Connection: swift.Connection{
124-
AuthUrl: cfg.Options.Openstack.Address,
125-
Region: cfg.Options.Openstack.Region,
126-
Tenant: cfg.Options.Openstack.Tenant,
127-
Domain: cfg.Options.Openstack.Domain,
128-
UserName: cfg.Options.Openstack.Username,
129-
ApiKey: cfg.Options.Openstack.Password,
130-
},
131-
containerprefix: cfg.Options.Openstack.ContainerPrefix,
132-
disableTempURL: cfg.Options.Openstack.DisableTempURL,
133-
projectIntegration: sdk.ProjectIntegration{Name: sdk.DefaultStorageIntegrationName},
134-
}
135-
if err := s.Authenticate(); err != nil {
136-
return nil, sdk.WrapError(err, "Unable to authenticate on swift storage")
137-
}
138-
return &s, nil
130+
return newSwiftStore(sdk.ProjectIntegration{Name: sdk.DefaultStorageIntegrationName}, cfg.Options.Openstack)
131+
case AWSS3:
132+
return newS3Store(sdk.ProjectIntegration{Name: sdk.DefaultStorageIntegrationName}, cfg.Options.AWSS3)
139133
case Filesystem:
140-
return newFilesystemStore(sdk.ProjectIntegration{Name: sdk.DefaultStorageIntegrationName}, cfg.Options.Filesystem.Basedir)
134+
return newFilesystemStore(sdk.ProjectIntegration{Name: sdk.DefaultStorageIntegrationName}, cfg.Options.Filesystem)
141135
default:
142136
return nil, fmt.Errorf("Invalid flag --artifact-mode")
143137
}

0 commit comments

Comments
 (0)
0