caicloud · caicloud-bot · Dec 3, 2018 · Nov 30, 2018 · Dec 3, 2018 · Dec 3, 2018
diff --git a/docs/api/v1/api.md b/docs/api/v1/api.md
@@ -44,6 +44,7 @@
     - [List SCM Repos](#list-scm-repos)
     - [List SCM Branches](#list-scm-branches)
     - [List SCM Tags](#list-scm-tags)
+    - [List SCM Dockerfiles](#list-scm-dockerfiles)
     - [Get SCM Templatetype](#get-scm-templatetype)
     - [Github webhook](#github-webhook)
     - [Gitlab webhook](#gitlab-webhook)
@@ -145,8 +146,9 @@
 | --- | --- | --- |
 | List | GET `/api/v1/projects/{project}/repos` | [link](#list-scm-repos) |
 | List | GET `/api/v1/projects/{project}/branches?repo=` | [link](#list-scm-branches) |
-| List | GET `/api/v1/projects/{project}/tags?repo=` | WIP, [link](#list-scm-tags) |
-| Get  | GET `/api/v1/projects/{project}/templatetype?repo=` | WIP, [link](#get-scm-templatetype) |
+| List | GET `/api/v1/projects/{project}/tags?repo=` | [link](#list-scm-tags) |
+| List | GET `/api/v1/projects/{project}/dockerfiles?repo=` | [link](#list-scm-dockerfiles) |
+| Get  | GET `/api/v1/projects/{project}/templatetype?repo=` | [link](#get-scm-templatetype) |
 
 ### Webhook API
 
@@ -1269,6 +1271,33 @@ Success:
 }
 ```
 
+### List SCM Dockerfiles
+
+List all dockerfiles for the repository.
+
+**Request**
+
+URL: `GET /api/v1/projects/{project}/dockerfiles?repo=`
+
+Note:
+
+| Field | Note |
+| --- | --- |
+| repo | Required, all dockerfiles in the repository will be listed. should in format of '{owner}/{repo}' |
+
+**Response**
+
+Success:
+
+```
+200 OK
+
+{
+    "metadata": <PaginationObject>,
+    "items": [ <dockerfile-path>, ... ]
+}
+```
+
 ### Get SCM Templatetype
 
 Get the template type of the specific repo.

diff --git a/pkg/api/v1/descriptor/project.go b/pkg/api/v1/descriptor/project.go
@@ -195,4 +195,26 @@ var projects = []definition.Descriptor{
 			},
 		},
 	},
+	{
+		Path: "/projects/{project}/dockerfiles",
+		Definitions: []definition.Definition{
+			{
+				Method:      definition.List,
+				Function:    handler.ListDockerfiles,
+				Description: "List dockerfiles' path of the repo for the project",
+				Parameters: []definition.Parameter{
+					{
+						Source: definition.Path,
+						Name:   httputil.ProjectPathParameterName,
+					},
+					{
+						Source:    definition.Query,
+						Name:      httputil.RepoQueryParameter,
+						Operators: []definition.Operator{validator.String("required")},
+					},
+				},
+				Results: definition.DataErrorResults("dockerfiles"),
+			},
+		},
+	},
 }
diff --git a/pkg/api/v1/handler/project.go b/pkg/api/v1/handler/project.go
@@ -138,6 +138,21 @@ func ListTags(ctx context.Context, name, repo string) (api.ListResponse, error)
 	return httputil.ResponseWithList(tags, len(tags)), nil
 }
 
+// ListDockerfiles handles the request to list dockerfiles for SCM repositories.
+func ListDockerfiles(ctx context.Context, name, repo string) (api.ListResponse, error) {
+	_, err := projectManager.GetProject(name)
+	if err != nil {
+		return api.ListResponse{}, err
+	}
+
+	dockerfiles, err := projectManager.ListDockerfiles(name, repo)
+	if err != nil {
+		return api.ListResponse{}, err
+	}
+
+	return httputil.ResponseWithList(dockerfiles, len(dockerfiles)), nil
+}
+
 // GetTemplateType handles the request to get project type for SCM repositories.
 func GetTemplateType(ctx context.Context, name, repo string) (*api.TemplateType, error) {
 	templateType := &api.TemplateType{}

diff --git a/pkg/scm/provider/github/github.go b/pkg/scm/provider/github/github.go
@@ -169,7 +169,7 @@ func (g *Github) ListBranches(repo string) ([]string, error) {
 	if strings.Contains(repo, "/") {
 		parts := strings.Split(repo, "/")
 		if len(parts) != 2 {
-			err := fmt.Errorf("repo %s is not correct, only supports one left slash", repo)
+			err := fmt.Errorf("invalid repo %s, must in format of '{owner}/{repo}'", repo)
 			log.Error(err.Error())
 			return nil, err
 		}
@@ -207,7 +207,7 @@ func (g *Github) ListTags(repo string) ([]string, error) {
 	if strings.Contains(repo, "/") {
 		parts := strings.Split(repo, "/")
 		if len(parts) != 2 {
-			err := fmt.Errorf("repo %s is not correct, only supports one left slash", repo)
+			err := fmt.Errorf("invalid repo %s, must in format of '{owner}/{repo}'", repo)
 			log.Error(err.Error())
 			return nil, err
 		}
@@ -235,10 +235,54 @@ func (g *Github) ListTags(repo string) ([]string, error) {
 	return tags, nil
 }
 
+// ListDockerfiles lists the dockerfiles for specified repo.
+func (g *Github) ListDockerfiles(repo string) ([]string, error) {
+	opt := &github.SearchOptions{
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	}
+
+	owner := g.scmCfg.Username
+	if strings.Contains(repo, "/") {
+		parts := strings.Split(repo, "/")
+		if len(parts) != 2 {
+			err := fmt.Errorf("invalid repo %s, must in format of '{owner}/{repo}'", repo)
+			log.Error(err.Error())
+			return nil, err
+		}
+		owner, repo = parts[0], parts[1]
+	}
+
+	q := fmt.Sprintf("FROM filename:Dockerfile repo:%s/%s", owner, repo)
+	var allCodeResult []github.CodeResult
+	for {
+		csr, resp, err := g.client.Search.Code(q, opt)
+		if err != nil {
+			return nil, err
+		}
+
+		allCodeResult = append(allCodeResult, csr.CodeResults...)
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+
+	crs := []string{}
+	for _, c := range allCodeResult {
+		if *c.Name == "Dockerfile" {
+			crs = append(crs, *c.Path)
+		}
+	}
+
+	return crs, nil
+}
+
 // CreateWebHook creates webhook for specified repo.
 func (g *Github) CreateWebHook(repoURL string, webHook *scm.WebHook) error {
 	if webHook == nil || len(webHook.Url) == 0 || len(webHook.Events) == 0 {
-		return fmt.Errorf("The webhook %v is not correct", webHook)
+		return fmt.Errorf("the webhook %v is not correct", webHook)
 	}
 
 	// Hook name must be passed as "web".
@@ -382,7 +426,7 @@ func (g *Github) GetTemplateType(repo string) (string, error) {
 	languages, r, err := g.client.Repositories.ListLanguages(owner, repo)
 	log.Error(r, err)
 	if err != nil {
-		log.Error("list language failed:%v", err)
+		log.Errorf("list language failed:%v", err)
 		return "", err
 	}
 
@@ -393,7 +437,7 @@ func (g *Github) GetTemplateType(repo string) (string, error) {
 		opt := &github.RepositoryContentGetOptions{}
 		_, directories, _, err := g.client.Repositories.GetContents(owner, repo, "", opt)
 		if err != nil {
-			log.Error("get contents failed:%v", err)
+			log.Errorf("get contents failed:%v", err)
 			return language, nil
 		}
 

diff --git a/pkg/scm/provider/gitlab/gitlabv3.go b/pkg/scm/provider/gitlab/gitlabv3.go
@@ -123,6 +123,16 @@ func (g *GitlabV3) ListTags(repo string) ([]string, error) {
 	return tagNames, nil
 }
 
+// ListDockerfiles lists the Dockerfiles for specified repo.
+func (g *GitlabV3) ListDockerfiles(repo string) ([]string, error) {
+	// List Dockerfiles in a project with gitlab v3 api is very inefficient.
+	// There is not a proper api can be used to do this with GitLab v3.
+	//
+	// FYI:
+	// https://stackoverflow.com/questions/25127695/search-filenames-with-gitlab-api
+	return nil, errors.ErrorNotImplemented.Error("list gitlab v3 dockerfiles")
+}
+
 // CreateWebHook creates webhook for specified repo.
 func (g *GitlabV3) CreateWebHook(repoURL string, webHook *scm.WebHook) error {
 	if webHook == nil || len(webHook.Url) == 0 || len(webHook.Events) == 0 {

diff --git a/pkg/scm/provider/gitlab/gitlabv4.go b/pkg/scm/provider/gitlab/gitlabv4.go
@@ -133,6 +133,43 @@ func (g *GitlabV4) ListTags(repo string) ([]string, error) {
 	return tagNames, nil
 }
 
+// ListDockerfiles lists the Dockerfiles for specified repo.
+func (g *GitlabV4) ListDockerfiles(repo string) ([]string, error) {
+	recursive := true
+	opt := &gitlab.ListTreeOptions{
+		Recursive: &recursive,
+		ListOptions: gitlab.ListOptions{
+			PerPage: 100,
+		},
+	}
+
+	treeNodes := []*gitlab.TreeNode{}
+	for {
+		treeNode, resp, err := g.client.Repositories.ListTree(repo, opt)
+		if err != nil {
+			log.Errorf("Fail to list dockerfile for %s", repo)
+			return nil, err
+		}
+
+		treeNodes = append(treeNodes, treeNode...)
+
+		if resp.NextPage == 0 {
+			break
+		}
+
+		opt.Page = resp.NextPage
+	}
+
+	files := []string{}
+	for _, t := range treeNodes {
+		if t.Type == "blob" && t.Name == "Dockerfile" {
+			files = append(files, t.Path)
+		}
+	}
+
+	return files, nil
+}
+
 // CreateWebHook creates webhook for specified repo.
 func (g *GitlabV4) CreateWebHook(repoURL string, webHook *scm.WebHook) error {
 	if webHook == nil || len(webHook.Url) == 0 || len(webHook.Events) == 0 {

diff --git a/pkg/scm/provider/svn/svn.go b/pkg/scm/provider/svn/svn.go
@@ -69,6 +69,10 @@ func (s *SVN) ListTags(repo string) ([]string, error) {
 	return nil, errors.ErrorNotImplemented.Error("list svn tags")
 }
 
+func (s *SVN) ListDockerfiles(repo string) ([]string, error) {
+	return nil, errors.ErrorNotImplemented.Error("list svn dockerfiles")
+}
+
 func (s *SVN) CheckToken() bool {
 	return true
 }

diff --git a/pkg/scm/scm.go b/pkg/scm/scm.go
@@ -52,6 +52,7 @@ type SCMProvider interface {
 	ListRepos() ([]api.Repository, error)
 	ListBranches(repo string) ([]string, error)
 	ListTags(repo string) ([]string, error)
+	ListDockerfiles(repo string) ([]string, error)
 	GetTemplateType(repo string) (string, error)
 	CheckToken() bool
 	NewTagFromLatest(tagName, description, commitID, url string) error

diff --git a/pkg/server/manager/project.go b/pkg/server/manager/project.go
@@ -43,6 +43,7 @@ type ProjectManager interface {
 	ListRepos(projectName string) ([]api.Repository, error)
 	ListBranches(projectName string, repo string) ([]string, error)
 	ListTags(projectName string, repo string) ([]string, error)
+	ListDockerfiles(projectName string, repo string) ([]string, error)
 	GetTemplateType(projectName string, repo string) (string, error)
 	GetStatistics(projectName string, start, end time.Time) (*api.PipelineStatusStats, error)
 }
@@ -232,6 +233,22 @@ func (m *projectManager) ListTags(projectName string, repo string) ([]string, er
 	return sp.ListTags(repo)
 }
 
+// ListDockerfiles lists the dockerfiles of the SCM repos authorized for the project.
+func (m *projectManager) ListDockerfiles(projectName string, repo string) ([]string, error) {
+	project, err := m.GetProject(projectName)
+	if err != nil {
+		return nil, err
+	}
+
+	scmConfig := project.SCM
+	sp, err := scm.GetSCMProvider(scmConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return sp.ListDockerfiles(repo)
+}
+
 // GetTemplateType get the template type of the SCM repos authorized for the project.
 func (m *projectManager) GetTemplateType(projectName string, repo string) (string, error) {
 	project, err := m.GetProject(projectName)

diff --git a/pkg/worker/stage/helper_test.go b/pkg/worker/stage/helper_test.go
@@ -92,22 +92,26 @@ func TestGenerateStageFinishLog(t *testing.T) {
 }
 
 func TestUpdateRecordStageStatus(t *testing.T) {
+	stages := &api.BuildStages{
+		CodeCheckout: &api.CodeCheckoutStage{},
+		Package:      &api.PackageStage{},
+	}
 	pr := &api.PipelineRecord{
 		StageStatus: &api.StageStatus{
 			CodeCheckout: &api.CodeCheckoutStageStatus{},
 			Package:      &api.GeneralStageStatus{},
 		},
 	}
 
-	if err := updateRecordStageStatus(pr, api.CodeCheckoutStageName, api.Running, nil); err == nil {
+	if err := updateRecordStageStatus(stages, pr, api.CodeCheckoutStageName, api.Running, nil); err == nil {
 		if pr.StageStatus.CodeCheckout.Status != api.Running {
 			t.Errorf("expect code checkout stage status to be Running, but get %s", pr.StageStatus.CodeCheckout.Status)
 		}
 	} else {
 		t.Errorf("expect error to be nil, but got %v", err)
 	}
 
-	if err := updateRecordStageStatus(pr, api.PackageStageName, api.Failed, fmt.Errorf("meets error")); err == nil {
+	if err := updateRecordStageStatus(stages, pr, api.PackageStageName, api.Failed, fmt.Errorf("meets error")); err == nil {
 		if pr.StageStatus.Package.Status != api.Failed {
 			t.Errorf("expect code checkout stage status to be Running, but get %s", pr.StageStatus.CodeCheckout.Status)
 		}

diff --git a/pkg/worker/stage/stage_test.go b/pkg/worker/stage/stage_test.go
@@ -67,7 +67,17 @@ func init() {
 		Project: &api.Project{
 			Registry: &registry,
 		},
-		Pipeline: &api.Pipeline{},
+		Pipeline: &api.Pipeline{
+			Build: &api.Build{
+				Stages: &api.BuildStages{
+					UnitTest:        &api.UnitTestStage{},
+					Package:         &api.PackageStage{},
+					CodeScan:        &api.CodeScanStage{},
+					IntegrationTest: &api.IntegrationTestStage{},
+					ImageRelease:    &api.ImageReleaseStage{},
+				},
+			},
+		},
 		PipelineRecord: &api.PipelineRecord{
 			PerformParams: &api.PipelinePerformParams{
 				Ref: "refs/heads/master",