-
Notifications
You must be signed in to change notification settings - Fork 1.1k
[WIP] store: extract artifact layers automatically #915 8000 0
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5340288
b30695f
8234314
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -689,6 +689,7 @@ | |
return err | ||
} | ||
|
||
c.Cleanup() | ||
c.state.Finished = time.Now() | ||
|
||
return nil | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,14 +1,18 @@ | ||
package ociartifact | ||
|
||
import ( | ||
"archive/tar" | ||
"compress/gzip" | ||
"context" | ||
"encoding/hex" | ||
"errors" | ||
"fmt" | ||
"io" | ||
"os" | ||
"path/filepath" | ||
"slices" | ||
"strings" | ||
"sync" | ||
|
||
modelSpec "github.com/CloudNativeAI/model-spec/specs-go/v1" | ||
"github.com/containers/common/libimage" | ||
|
@@ -17,6 +21,7 @@ | |
"github.com/containers/image/v5/oci/layout" | ||
"github.com/containers/image/v5/pkg/blobinfocache" | ||
"github.com/containers/image/v5/types" | ||
"github.com/klauspost/compress/zstd" | ||
"github.com/opencontainers/go-digest" | ||
specs "github.com/opencontainers/image-spec/specs-go/v1" | ||
|
||
|
@@ -469,7 +474,16 @@ | |
// Source path of the blob, i.e. full path in the blob dir. | ||
SourcePath string | ||
// Name of the file in the artifact. | ||
Name string | ||
Name string | ||
cleanupFunc func() | ||
cleanupOnce *sync.Once | ||
} | ||
|
||
// Cleanup triggers the deletion of the extracted directory. | ||
func (b *BlobMountPath) Cleanup() { | ||
if b.cleanupFunc != nil && b.cleanupOnce != nil { | ||
b.cleanupOnce.Do(b.cleanupFunc) | ||
} | ||
} | ||
|
||
// BlobMountPaths retrieves the local file paths for all blobs in the provided artifact and returns them as BlobMountPath slices. | ||
|
@@ -479,35 +493,131 @@ | |
return nil, fmt.Errorf("failed to get an image reference: %w", err) | ||
} | ||
|
||
extractDir := filepath.Join(s.rootPath, "extracted", artifact.Digest().Encoded()) | ||
if err := os.MkdirAll(extractDir, 0o755); err != nil { | ||
return nil, fmt.Errorf("failed to create artifact dir: %w", err) | ||
} | ||
|
||
cleanupFunc := func() { os.RemoveAll(extractDir) } | ||
cleanupOnce := &sync.Once{} | ||
|
||
// Cleanup on function exit if we encounter an error | ||
cleanup := true | ||
defer func() { | ||
if cleanup { | ||
os.RemoveAll(extractDir) | ||
} | ||
}() | ||
|
||
src, err := ref.NewImageSource(ctx, sys) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to get an image source: %w", err) | ||
} | ||
|
||
defer src.Close() | ||
|
||
mountPaths := make([]BlobMountPath, 0, len(artifact.Manifest().Layers)) | ||
bic := blobinfocache.DefaultCache(s.systemContext) | ||
|
||
for _, l := range artifact.Manifest().Layers { | ||
path, err := layout.GetLocalBlobPath(ctx, src, l.Digest) | ||
err = func() error { | ||
blob, _, err := s.impl.GetBlob(ctx, src, types.BlobInfo{Digest: l.Digest}, bic) | ||
if err != nil { | ||
return fmt.Errorf("failed to get blob: %w", err) | ||
} | ||
defer blob.Close() | ||
|
||
content, err := s.processLayerContent(l.MediaType, blob) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder which is better extracting when pulling or when mounting. |
||
if err != nil { | ||
return fmt.Errorf("failed to process layer: %w", err) | ||
} | ||
|
||
name := artifactName(l.Annotations) | ||
if name == "" { | ||
log.Warnf(ctx, "Unable to find name for artifact layer which makes it not mountable") | ||
|
||
return nil | ||
} | ||
|
||
filePath := filepath.Join(extractDir, name) | ||
if err := os.WriteFile(filePath, content, 0o644); err != nil { | ||
log.Errorf(ctx, "Failed to write file: %v", err) | ||
os.Remove(filePath) | ||
return err | ||
} | ||
|
||
mountPaths = append(mountPaths, BlobMountPath{ | ||
SourcePath: filePath, | ||
Name: name, | ||
cleanupFunc: cleanupFunc, | ||
cleanupOnce: cleanupOnce, | ||
}) | ||
return nil | ||
Check failure on line 554 in internal/ociartifact/store.go
|
||
}() | ||
// If any layer extraction fails, abort the entire operation. | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to extract artifact layer: %w", err) | ||
} | ||
} | ||
// Disable cleanup on success. | ||
// mountArtifact() will handle the cleanup. | ||
cleanup = false | ||
return mountPaths, nil | ||
Check failure on line 564 in internal/ociartifact/store.go
|
||
} | ||
|
||
// processLayerContent decompresses and processes layer content based on media type. | ||
// Supports gzip, zstd, and uncompressed tar formats. Returns raw bytes for non-tar content. | ||
// Instead of relying on the exact media type like "application/vnd.oci.image.layer.v1.tar+zstd", | ||
// it would be helpful to stick with more generic patterns like ".tar+gzip" or ".tar+zstd". | ||
func (s *Store) processLayerContent(mediaType string, r io.Reader) ([]byte, error) { | ||
switch { | ||
case strings.HasSuffix(mediaType, ".tar+gzip"): | ||
gr, err := gzip.NewReader(r) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to get a local blob path: %w", err) | ||
return nil, fmt.Errorf("gzip decompress failed: %w", err) | ||
} | ||
defer gr.Close() | ||
|
||
name := artifactName(l.Annotations) | ||
if name == "" { | ||
log.Warnf(ctx, "Unable to find name for artifact layer which makes it not mountable") | ||
return s.extractSingleFileFromTar(gr) | ||
|
||
continue | ||
case strings.HasSuffix(mediaType, ".tar+zstd"): | ||
zr, err := zstd.NewReader(r) | ||
if err != nil { | ||
return nil, fmt.Errorf("zstd decompress failed: %w", err) | ||
} | ||
defer zr.Close() | ||
|
||
mountPaths = append(mountPaths, BlobMountPath{ | ||
SourcePath: path, | ||
Name: name, | ||
}) | ||
return s.extractSingleFileFromTar(zr) | ||
|
||
case strings.HasSuffix(mediaType, ".tar"): | ||
return s.extractSingleFileFromTar(r) | ||
|
||
default: | ||
data, err := io.ReadAll(r) | ||
if err != nil { | ||
return nil, fmt.Errorf("reading blob content: %w", err) | ||
} | ||
|
||
return data, nil | ||
} | ||
} | ||
|
||
return mountPaths, nil | ||
func (s *Store) extractSingleFileFromTar(r io.Reader) ([]byte, error) { | ||
tr := tar.NewReader(r) | ||
|
||
for { | ||
hdr, err := tr.Next() | ||
if err == io.EOF { | ||
return nil, errors.New("no files found in tar archive") | ||
} | ||
|
||
if err != nil { | ||
return nil, fmt.Errorf("reading tar header: %w", err) | ||
} | ||
|
||
if hdr.Typeflag == tar.TypeReg { | ||
return io.ReadAll(tr) | ||
} | ||
} | ||
} | ||
|
||
func artifactName(annotations map[string]string) string { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
unsure about this. is this a convention? is there a convention? seems odd to hardcode the path but I'm unsure about the alternative
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah, I tried changing that logic. Let me know how it feels.3c63062