8000 Cache more files in the filesystem when reading a pack file by samoht · Pull Request #163 · mirage/ocaml-git · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Cache more files in the filesystem when reading a pack file #163

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 4, 2016
Merged

Cache more files in the filesystem when reading a pack file #163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions lib/git.mli
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -752,14 +752,16 @@ module Packed_value: sig
buffer. *)

val to_value:
index:Pack_index.f -> read:Value.read_inflated -> version:int ->
index:Pack_index.f ->
read:Value.read_inflated -> write:Value.write_inflated -> version:int ->
ba:Cstruct.buffer -> t -> Value.t Lwt.t
(** Unpack the packed value using the provided indexes. *)

(** {2 Position independant packed values} *)

val unpack:
index:Pack_index.f -> read:Value.read_inflated -> version:int ->
index:Pack_index.f ->
read:Value.read_inflated -> write:Value.write_inflated -> version:int ->
ba:Cstruct.buffer -> t -> string Lwt.t
(** Same as {!to_value} but for inflated raw buffers. *)

Expand Down Expand Up @@ -870,11 +872,13 @@ module Pack: sig
(** Unpack a whole pack file on disk (by calling [write] on every
values) and return the Hash1s of the written objects. *)

val read: index:Pack_index.f -> read:Value.read_inflated ->
val read: index:Pack_index.f ->
read:Value.read_inflated -> write:Value.write_inflated ->
Mstruct.t -> Hash.t -> Value.t option Lwt.t
(** Same as the top-level [read] function but for raw packs. *)

val read_inflated: index:Pack_index.f -> read:Value.read_inflated ->
val read_inflated: index:Pack_index.f ->
read:Value.read_inflated -> write:Value.write_inflated ->
Mstruct.t -> Hash.t -> string option Lwt.t
(** Same as {!read} but for inflated values. *)

Expand Down
41 changes: 24 additions & 17 deletions lib/git_fs.ml
View file
Open in desktop
8000
Original file line number Diff line number Diff line change
Expand Up @@ -388,9 +388,14 @@ module Make (IO: IO) (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
| None -> read_in_pack t pack h
) None packs

let read t h ~read = read_aux (read_in_pack "read" (Pack_IO.Raw.read ~read)) t h
let read_inflated t h ~read =
read_aux (read_in_pack "read_inflated" (Pack_IO.Raw.read_inflated ~read)) t h
let read t h ~read ~write =
read_aux (read_in_pack "read" (Pack_IO.Raw.read ~read ~write)) t h

let read_inflated t h ~read ~write =
read_aux
(read_in_pack "read_inflated" (Pack_IO.Raw.read_inflated ~read ~write))
t h

let size = read_aux (read_in_pack "size" Pack_IO.Raw.size)

let mem t h =
Expand Down Expand Up @@ -418,6 +423,18 @@ module Make (IO: IO) (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
| None -> None
| Some v -> Git_value.Cache.add_inflated h v; Some v

let write t value =
Loose.write t value >>= fun h ->
Log.debug (fun l -> l "write -> %a" Git_hash.pp h);
Git_value.Cache.add h value;
Lwt.return h

let write_inflated t value =
Loose.write_inflated t value >>= fun h ->
Log.debug (fun l -> l "write -> %a" Git_hash.pp h);
Git_value.Cache.add_inflated h value;
Lwt.return h

let rec read t h =
Log.debug (fun l -> l "read %a" Git_hash.pp h);
match Git_value.Cache.find h with
Expand All @@ -429,7 +446,8 @@ module Make (IO: IO) (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
| Some v -> Lwt.return (Some v)
| None ->
let read = read_inflated t in
Packed.read ~read t h
let write = write_inflated t in
Packed.read ~read ~write t h
end >|=
cache_add h

Expand All @@ -444,7 +462,8 @@ module Make (IO: IO) (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
| Some v -> Lwt.return (Some v)
| None ->
let read = read_inflated t in
Packed.read_inflated ~read t h
let write = write_inflated t in
Packed.read_inflated ~read ~write t h
end >|=
cache_add_inflated h

Expand Down Expand Up @@ -566,18 +585,6 @@ module Make (IO: IO) (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
| None ->
err_not_found "read_reference_exn" (Fmt.to_to_string Git_reference.pp r)

let write t value =
Loose.write t value >>= fun h ->
Log.debug (fun l -> l "write -> %a" Git_hash.pp h);
Git_value.Cache.add h value;
Lwt.return h

let write_inflated t value =
Loose.write_inflated t value >>= fun h ->
Log.debug (fun l -> l "write -> %a" Git_hash.pp h);
Git_value.Cache.add_inflated h value;
Lwt.return h

let write_pack t pack =
Log.debug (fun l -> l "write_pack");
let name = Git_pack.Raw.name pack in
Expand Down
14 changes: 8 additions & 6 deletions lib/git_pack.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,18 +143,18 @@ module IO (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
let v = Git_packed_value.create ~kind ~offset:shift in
Some (version, ba, v)

let read ~index ~read buf hash =
let read ~index ~read ~write buf hash =
match read_packed_value "read" ~index buf hash with
| None -> Lwt.return_none
| Some (version, ba, v) ->
Packed_value_IO.to_value ~version ~read ~index ~ba v >|=
Packed_value_IO.to_value ~version ~read ~write ~index ~ba v >|=
fun x -> Some x

let read_inflated ~index ~read buf hash =
let read_inflated ~index ~read ~write buf hash =
match read_packed_value "read_inflated" ~index buf hash with
| None -> Lwt.return_none
| Some (version, ba, v) ->
Packed_value_IO.unpack ~version ~read ~index ~ba v >|=
Packed_value_IO.unpack ~version ~read ~write ~index ~ba v >|=
fun x -> Some x

let size ~index buf hash =
Expand Down Expand Up @@ -433,9 +433,11 @@ module type IO = sig
Mstruct.t -> t Lwt.t
val unpack: ?progress:(string -> unit) -> write:Git_value.write_inflated ->
t -> Git_hash.Set.t Lwt.t
val read: index:Git_pack_index.f -> read:Git_value.read_inflated ->
val read: index:Git_pack_index.f ->
read:Git_value.read_inflated -> write:Git_value.write_inflated ->
Mstruct.t -> Git_hash.t -> Git_value.t option Lwt.t
val read_inflated: index:Git_pack_index.f -> read:Git_value.read_inflated ->
val read_inflated: index:Git_pack_index.f ->
read:Git_value.read_inflated -> write:Git_value.write_inflated ->
Mstruct.t -> Git_hash.t -> string option Lwt.t
val size: index:Git_pack_index.f -> Mstruct.t -> Git_hash.t -> int option Lwt.t
end
Expand Down
6 changes: 4 additions & 2 deletions lib/git_pack.mli
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,11 @@ module type IO = sig
Mstruct.t -> t Lwt.t
val unpack: ?progress:(string -> unit) -> write:Git_value.write_inflated ->
t -> Git_hash.Set.t Lwt.t
val read: index:Git_pack_index.f -> read:Git_value.read_inflated ->
val read: index:Git_pack_index.f ->
read:Git_value.read_inflated -> write:Git_value.write_inflated ->
Mstruct.t -> Git_hash.t -> Git_value.t option Lwt.t
val read_inflated: index:Git_pack_index.f -> read:Git_value.read_inflated ->
val read_inflated: index:Git_pack_index.f ->
read:Git_value.read_inflated -> write:Git_value.write_inflated ->
Mstruct.t -> Git_hash.t -> string option Lwt.t
val size: index:Git_pack_index.f -> Mstruct.t -> Git_hash.t -> int option Lwt.t
end
Expand Down
30 changes: 16 additions & 14 deletions lib/git_packed_value.ml
View file
Open in desktop 6D40
Original file line number Diff line number Diff line change
Expand Up @@ -538,7 +538,7 @@ module IO (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
| 3 -> V3.input
| _ -> fail "pack version should be 2 or 3"

let rec unpack_ref_delta ~lv ~version ~index ~read ba d =
let rec unpack_ref_delta ~lv ~version ~index ~read ~write ba d =
Log.debug (fun l ->
l "unpack-ref-delta[%d]: d.source=%a" lv Git_hash.pp d.source);
match index d.source with
Expand All @@ -549,14 +549,14 @@ module IO (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
let buf = Mstruct.of_bigarray ~off:offset ~len ba in
let kind = input_packed_value version buf in
let t = { offset; kind } in
unpack ~lv:(lv+1) ~read ~version ~index ~ba t >|= fun source ->
unpack ~lv:(lv+1) ~read ~write ~version ~index ~ba t >|= fun source ->
Git_misc.with_buffer (fun b -> add_delta b {d with source})
| None ->
read d.source >>= function
| None -> fail "unpack: cannot read %a" Git_hash.pp d.source
| Some buf -> Lwt.return buf

and unpack_off_delta ~lv ~version ~index ~read ba d offset =
and unpack_off_delta ~lv ~version ~index ~read ~write ba d offset =
let offset = offset - d.source in
Log.debug (fun l ->
l "unpack-off-delta[%d]: offset=%d-%d=%d" lv offset d.source offset);
Expand All @@ -565,23 +565,25 @@ module IO (D: Git_hash.DIGEST) (I: Git_inflate.S) = struct
let buf = Mstruct.of_bigarray ~off:offset ~len ba in
let kind = input_packed_value version buf in
let t = { offset; kind } in
unpack ~lv:(lv+1) ~read ~version ~index ~ba t >|= fun source ->
unpack ~lv:(lv+1) ~read ~write ~version ~index ~ba t >|= fun source ->
Git_misc.with_buffer (fun b -> add_delta b {d with source})

and unpack ~lv ~index ~read ~version ~ba { offset; kind } =
match kind with
| Raw_value x ->
Log.debug (fun l -> l "unpack-raw[%d]" lv); Lwt.return x
| Ref_delta d ->
unpack_ref_delta ~lv ~version ~index ~read ba d
| Off_delta d ->
unpack_off_delta ~lv ~version ~index ~read ba d offset
and unpack ~lv ~index ~read ~write ~version ~ba { offset=o; kind } =
let obj = match kind with
| Raw_value v -> Lwt.return v
| Ref_delta d -> unpack_ref_delta ~lv ~version ~index ~read ~write ba d
| Off_delta d -> unpack_off_delta ~lv ~version ~index ~read ~write ba d o
in
obj >>= fun x ->
write x >|= fun h ->
Log.debug (fun l -> l "unpack-raw[%d]: %a" lv Git_hash.pp h);
x

let unpack = unpack ~lv:0

let to_value ~index ~read ~version ~ba t =
let to_value ~index ~read ~write ~version ~ba t =
Log.debug (fun l -> l "to_value");
unpack ~version ~read ~index ~ba t >|= fun u ->
unpack ~version ~read ~write ~index ~ba t >|= fun u ->
(* FIXME: costly, allocates a bigarray *)
Value_IO.input_inflated (Mstruct.of_string u)

Expand Down
10 changes: 6 additions & 4 deletions lib/git_packed_value.mli
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,14 @@ module IO (D: Git_hash.DIGEST) (I: Git_inflate.S): sig
Buffer.t -> t -> unit Lwt.t

val to_value:
index:Git_pack_index.f -> read:Git_value.read_inflated -> version:int ->
ba:Cstruct.buffer -> t -> Git_value.t Lwt.t
index:Git_pack_index.f ->
read:Git_value.read_inflated -> write:Git_value.write_inflated ->
version:int -> ba:Cstruct.buffer -> t -> Git_value.t Lwt.t

val unpack:
index:Git_pack_index.f -> read:Git_value.read_inflated -> version:int ->
ba:Cstruct.buffer -> t -> string Lwt.t
index:Git_pack_index.f ->
read:Git_value.read_inflated -> write:Git_value.write_inflated ->
version:int -> ba:Cstruct.buffer -> t -> string Lwt.t

val value_of_pic: pic -> Git_value.t

Expand Down
0