diff --git a/build.sbt b/build.sbt index 25a330f..0e76294 100644 --- a/build.sbt +++ b/build.sbt @@ -1,20 +1,12 @@ -resolvers += "Java.net Maven2 Repository" at "http://download.java.net/maven/2/" +libraryDependencies += "org.scala-lang.modules" %% "scala-async" % "0.9.1" -resolvers += "apache" at "https://repository.apache.org/content/repositories/snapshots/" +libraryDependencies += "net.databinder.dispatch" %% "dispatch-core" % "0.11.1" -resolvers += "typesafe" at "http://repo.typesafe.com/typesafe/snapshots/" +libraryDependencies += "net.databinder.dispatch" %% "dispatch-lift-json" % "0.11.0" -resolvers ++= Seq("snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", - "releases" at "http://oss.sonatype.org/content/repositories/releases") +libraryDependencies += "org.specs2" %% "specs2" % "1.12.3" % "test" -libraryDependencies ++= Seq( - "net.databinder.dispatch" %% "dispatch-core" % "0.11.0", - "net.databinder.dispatch" %% "dispatch-lift-json" % "0.11.0", - "org.specs2" %% "specs2" % "1.12.3" % "test" -// "org.slf4j" % "slf4j-api" % "1.7.2", -// "org.slf4j" % "slf4j-simple" % "1.7.2", -// "ch.qos.logback" % "logback-core" % "1.0.6" -) +libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2" parallelExecution in Test := false @@ -23,3 +15,5 @@ name := "dispatch-github" organization := "dispatch" version := "0.1-SNAPSHOT" + +scalaVersion := "2.10.4" \ No newline at end of file diff --git a/src/main/scala/dispatch/github/Agent.scala b/src/main/scala/dispatch/github/Agent.scala new file mode 100644 index 0000000..60534b4 --- /dev/null +++ b/src/main/scala/dispatch/github/Agent.scala @@ -0,0 +1,84 @@ +package dispatch.github + +import scala.concurrent._ +import scala.concurrent.duration._ +import scala.async.Async.{async, await} +import dispatch.{Req, Http, as, url} +import net.liftweb.json._ +import com.ning.http.client.Response +import com.typesafe.scalalogging.slf4j.LazyLogging + +trait Client extends LazyLogging { + + def searchUsers(q : String) = new UserSearch(this, q) + + def searchCode(q : String) = new CodeSearch(this, q) + + def searchRepos(q : String) = new RepoSearch(this, q) + + private[github] def pause(resp : Response)(implicit ec : ExecutionContext) + : Future[Boolean] = Future { + val hdrs = resp.getHeaders() + hdrs.getFirstValue("X-RateLimit-Remaining") match { + case "0" => { + val reset = 1000 * hdrs.getFirstValue("X-RateLimit-Reset").toLong + val now = System.currentTimeMillis() + // An extra second for clock skew. + val delay = math.max(1000, 1000 + reset - now) + logger.debug(s"GitHub API rate limit reached: pausing for ${delay / 1000}ms") + Thread.sleep(delay) + true + } + case _ => false + } + } + + def request(req : Req) + (implicit ec : ExecutionContext) : Future[Response] = async { + val resp = await(Http(req)) + resp.getStatusCode() match { + case 403 => { + if (await(pause(resp))) { + await(request(req)) + } + else { + resp + } + } + case _ => { + await(pause(resp)) + resp + } + } + } + + private[github] def searchByUrl[T](req : Req) + (implicit m : Manifest[T], ec : ExecutionContext) + : Future[Stream[T]] = async { + implicit val formats = DefaultFormats + + val resp = await(this.request(req)) + val json = as.lift.Json(resp) + val results = Stream((json \\ "items").extract[List[T]] :_*) + nextPage(resp) match { + case None => results + case Some(nextUrl) => { + lazy val rest = Await.result(searchByUrl(url(nextUrl)), Duration.Inf) + results.append(rest) + } + } + } + +} + +class OAuthClient(accessToken : String) extends Client { + + override def request(req : Req) + (implicit ec : ExecutionContext) : Future[Response] = + super.request(req.addHeader("Authorizaton", "token " + accessToken)) + +} + +object BasicClient extends Client { + +} diff --git a/src/main/scala/dispatch/github/GhRepository.scala b/src/main/scala/dispatch/github/GhRepository.scala index 425fbd4..0fa38a1 100644 --- a/src/main/scala/dispatch/github/GhRepository.scala +++ b/src/main/scala/dispatch/github/GhRepository.scala @@ -8,8 +8,10 @@ import java.text.SimpleDateFormat case class GhRSimpleRepository(id: Int, owner: GhOwner, name: String, html_url: String, description: String) -case class GhRepository(id: Int, owner: GhOwner, name: String, updated_at: Date, language: String, - html_url: String, clone_url: String, description: String, open_issues: Int) +case class GhRepository(id: Int, full_name : String, owner: GhOwner, + name: String, updated_at: Date, language: String, + html_url: String, clone_url: String, + description: String, open_issues: Int) case class GhBranchSummary(name: String, commit: GhCommitId) diff --git a/src/main/scala/dispatch/github/GhSearch.scala b/src/main/scala/dispatch/github/GhSearch.scala index c8cb716..28dc232 100644 --- a/src/main/scala/dispatch/github/GhSearch.scala +++ b/src/main/scala/dispatch/github/GhSearch.scala @@ -1,50 +1,67 @@ package dispatch.github -import scala.concurrent.Future +import scala.concurrent._ +import scala.concurrent.duration._ import dispatch._ -import Defaults._ -import net.liftweb.json._ - -sealed abstract class Sort -case class Stars() extends Sort -case class BestMatch() extends Sort -case class Forks() extends Sort -case class Updated() extends Sort +import com.ning.http.client.Response case class GhCode(name : String, path : String, sha : String, url : String, git_url : String, html_url : String, score : Double, repository : GhRSimpleRepository) -object GhSearch { +class UserSearch private(client : Client, params : Map[String, String]) { + + def this(client : Client, q : String) = this(client, Map("q" -> q)) + + def ascending() = new UserSearch(client, params + ("sort" -> "asc")) + + def byFollowers() = new UserSearch(client, params + ("order" -> "followers")) + + def byRepositories() = + new UserSearch(client, params + ("order" -> "repositories")) + + def byJoined() = new UserSearch(client, params + ("order" -> "joined")) + + def perPage(n : Int) = + new UserSearch(client, params + ("per_page" -> n.toString)) + + def search()(implicit ec : ExecutionContext) = { + val url = (GitHub.api_host / "search" / "users" < query, - "order" -> (if (ascending) "asc" else "desc")) - val params = sort match { - case BestMatch() => params1 - case Stars() => params1 + ("order" -> "stars") - case Updated() => params1 + ("order" -> "updated") - case Forks() => params1 + ("order" -> "forks") - } + def this(client : Client, q : String) = this(client, Map("q" -> q)) - val respJson = Http(svc.secure < "indexed")) + + def ascending() = new CodeSearch(client, params + ("sort" -> "asc")) + + def search()(implicit ec : ExecutionContext) = { + val url = (GitHub.api_host / "search" / "code" < query, - "order" -> (if (ascending) "asc" else "desc")) - val params = - if (sortByIndexed) params1 + ("order" -> "indexed") else params1 - - val respJson = Http(svc.secure < q)) + + def byStars() = new RepoSearch(client, params + ("order" -> "stars")) + + def byForks() = new RepoSearch(client, params + ("order" -> "forks")) + + def byUpdated() = new RepoSearch(client, params + ("order" -> "updated")) + + def ascending() = new RepoSearch(client, params + ("sort" -> "asc")) + + def search()(implicit ec : ExecutionContext) = { + val url = (GitHub.api_host / "search" / "repositories" <; rel="next".*)""".r + + /** Returns a URL to the next page of results, if present in the "Link" + * header. + * + * https://developer.github.com/v3/#pagination + */ + private[github] def nextPage(resp : Response) : Option[String] = + resp.getHeaders().getFirstValue("Link") match { + case null => None + case str => nextPageRegex.findFirstIn(str) + } + +} \ No newline at end of file