Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package cmd

import (
"log"
"net/http"
_ "net/http/pprof"
"os"
Expand Down Expand Up @@ -51,7 +50,7 @@ var rootCmd = &cobra.Command{
go func() {
err := http.ListenAndServe(rootConfig.PprofAddr, nil)
if err != nil {
log.Fatal(err)
logrus.Fatal(err)
}
}()
}
Expand Down
2 changes: 1 addition & 1 deletion internal/pb/pb.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ func (p *ProgressBar) Abort(name string, err error) {
p.mu.RUnlock()

if ok {
logrus.Errorf("abort the progress bar[%s] as error occurred: %v", name, err)
logrus.Errorf("progress: aborting bar %s: %v", name, err)
bar.Abort(true)
}
}
Expand Down
10 changes: 5 additions & 5 deletions pkg/backend/attach.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ var (

// Attach attaches user materials into the model artifact which follows the Model Spec.
func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attach) error {
logrus.Infof("attach: starting attach operation for file %s [config: %+v]", filepath, cfg)
logrus.Infof("attach: attaching file %s", filepath)
srcManifest, err := b.getManifest(ctx, cfg.Source, cfg.OutputRemote, cfg.PlainHTTP, cfg.Insecure)
if err != nil {
return fmt.Errorf("failed to get source manifest: %w", err)
Expand All @@ -73,7 +73,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
return fmt.Errorf("failed to get source model config: %w", err)
}

logrus.Infof("attach: loaded source model config [%+v]", srcModelConfig)
logrus.Infof("attach: loaded source model config [config: %+v]", srcModelConfig)

proc := b.getProcessor(cfg.DestinationDir, filepath, cfg.Raw)
if proc == nil {
Expand Down Expand Up @@ -111,7 +111,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
}
}

logrus.Infof("attach: found existing layer for file %s [%+v]", filepath, foundLayer)
logrus.Infof("attach: found existing layer for file %s [layer: %+v]", filepath, foundLayer)
if foundLayer != nil {
// Remove the found layer from the layers slice as we need to replace it with the new layer.
for i, layer := range layers {
Expand Down Expand Up @@ -177,7 +177,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
}
}

logrus.Infof("attach: built model config [%+v]", config)
logrus.Infof("attach: built model config [config: %+v]", config)

configDesc, err := builder.BuildConfig(ctx, config, hooks.NewHooks(
hooks.WithOnStart(func(name string, size int64, reader io.Reader) io.Reader {
Expand Down Expand Up @@ -210,7 +210,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
return fmt.Errorf("failed to build model manifest: %w", err)
}

logrus.Infof("attach: successfully attached file %s", filepath)
logrus.Infof("attach: attached file %s", filepath)
return nil
}

Expand Down
6 changes: 3 additions & 3 deletions pkg/backend/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ const (

// Build builds the user materials into the model artifact which follows the Model Spec.
func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target string, cfg *config.Build) error {
logrus.Infof("build: starting build operation for target %s [config: %+v]", target, cfg)
logrus.Infof("build: building artifact %s", target)
// parse the repo name and tag name from target.
ref, err := ParseReference(target)
if err != nil {
Expand Down Expand Up @@ -95,7 +95,7 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri

layers = append(layers, layerDescs...)

logrus.Infof("build: processed layers for artifact [count: %d, layers: %+v]", len(layers), layers)
logrus.Infof("build: processed layers [count: %d, layers: %+v]", len(layers), layers)

revision := sourceInfo.Commit
if revision != "" && sourceInfo.Dirty {
Expand Down Expand Up @@ -158,7 +158,7 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
return fmt.Errorf("failed to build model manifest: %w", err)
}

logrus.Infof("build: successfully built model artifact %s", target)
logrus.Infof("build: built artifact %s", target)
return nil
}

Expand Down
9 changes: 6 additions & 3 deletions pkg/backend/build/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func NewBuilder(outputType OutputType, store storage.Storage, repo, tag string,
cache, err := cache.New(os.TempDir())
if err != nil {
// Just print the error message because cache is not critical.
logrus.Errorf("failed to create cache: %v", err)
logrus.Errorf("builder: failed to create cache: %v", err)
}

return &abstractBuilder{
Expand Down Expand Up @@ -302,7 +302,7 @@ func (ab *abstractBuilder) retrieveCache(ctx context.Context, path string, info
return "", 0, false
}

logrus.Infof("builder: retrieved from cache for file %s [digest: %s]", path, item.Digest)
logrus.Infof("builder: cache hit for file %s [digest: %s]", path, item.Digest)
return item.Digest, item.Size, true
}

Expand Down Expand Up @@ -397,7 +397,10 @@ func addFileMetadata(desc *ocispec.Descriptor, path, relPath string) error {
if err != nil {
return fmt.Errorf("failed to marshal metadata: %w", err)
}
logrus.Infof("builder: retrieved metadata for file %s [metadata: %s]", relPath, string(metadataStr))
logrus.Infof(
"builder: retrieved metadata for file %s [metadata: %s]",
relPath, string(metadataStr),
)

if desc.Annotations == nil {
desc.Annotations = make(map[string]string)
Expand Down
11 changes: 7 additions & 4 deletions pkg/backend/extract.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ const (

// Extract extracts the model artifact.
func (b *backend) Extract(ctx context.Context, target string, cfg *config.Extract) error {
logrus.Infof("extract: starting extract operation for target %s [config: %+v]", target, cfg)
logrus.Infof("extract: extracting artifact %s", target)
// parse the repository and tag from the target.
ref, err := ParseReference(target)
if err != nil {
Expand Down Expand Up @@ -71,7 +71,7 @@ func exportModelArtifact(ctx context.Context, store storage.Storage, manifest oc
g, ctx := errgroup.WithContext(ctx)
g.SetLimit(cfg.Concurrency)

logrus.Infof("extract: processing layers for target %s [count: %d]", repo, len(manifest.Layers))
logrus.Infof("extract: extracting %d layers for %s", len(manifest.Layers), repo)
for _, layer := range manifest.Layers {
g.Go(func() error {
select {
Expand All @@ -91,7 +91,10 @@ func exportModelArtifact(ctx context.Context, store storage.Storage, manifest oc
bufferedReader := bufio.NewReaderSize(reader, defaultBufferSize)
if err := extractLayer(layer, cfg.Output, bufferedReader); err != nil {
if errors.Is(err, pkgcodec.ErrAlreadyUpToDate) {
logrus.Debugf("extract: skipped layer %s because target is up-to-date", layer.Digest.String())
logrus.Debugf(
"extract: skipping layer %s, already up-to-date",
layer.Digest.String(),
)
return nil
}

Expand All @@ -108,7 +111,7 @@ func exportModelArtifact(ctx context.Context, store storage.Storage, manifest oc
return err
}

logrus.Infof("extract: successfully extracted model artifact %s", repo)
logrus.Infof("extract: extracted artifact %s", repo)
return nil
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/backend/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ import (

// Fetch fetches partial files to the output.
func (b *backend) Fetch(ctx context.Context, target string, cfg *config.Fetch) error {
logrus.Infof("fetch: starting fetch operation for target %s [config: %+v]", target, cfg)
logrus.Infof("fetch: fetching from %s", target)

// fetchByDragonfly is called if a Dragonfly endpoint is specified in the configuration.
if cfg.DragonflyEndpoint != "" {
logrus.Infof("fetch: using dragonfly for target %s", target)
logrus.Infof("fetch: using dragonfly for %s", target)
return b.fetchByDragonfly(ctx, target, cfg)
}

Expand Down Expand Up @@ -104,7 +104,7 @@ func (b *backend) Fetch(ctx context.Context, target string, cfg *config.Fetch) e
g, ctx := errgroup.WithContext(ctx)
g.SetLimit(cfg.Concurrency)

logrus.Infof("fetch: processing matched layers [count: %d]", len(layers))
logrus.Infof("fetch: fetching %d matched layers", len(layers))
for _, layer := range layers {
g.Go(func() error {
select {
Expand All @@ -127,6 +127,6 @@ func (b *backend) Fetch(ctx context.Context, target string, cfg *config.Fetch) e
return err
}

logrus.Infof("fetch: successfully fetched layers [count: %d]", len(layers))
logrus.Infof("fetch: fetched %d layers", len(layers))
return nil
}
6 changes: 3 additions & 3 deletions pkg/backend/fetch_by_d7y.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ import (

// fetchByDragonfly fetches partial files via Dragonfly gRPC service based on pattern matching.
func (b *backend) fetchByDragonfly(ctx context.Context, target string, cfg *config.Fetch) error {
logrus.Infof("fetch: starting dragonfly fetch operation for target %s", target)
logrus.Infof("fetch: fetching from %s via dragonfly", target)

// Parse reference and initialize remote client.
ref, err := ParseReference(target)
Expand Down Expand Up @@ -127,7 +127,7 @@ func (b *backend) fetchByDragonfly(ctx context.Context, target string, cfg *conf
g, ctx := errgroup.WithContext(ctx)
g.SetLimit(cfg.Concurrency)

logrus.Infof("fetch: processing matched layers via dragonfly [count: %d]", len(layers))
logrus.Infof("fetch: fetching %d matched layers via dragonfly", len(layers))
for _, layer := range layers {
g.Go(func() error {
select {
Expand All @@ -149,7 +149,7 @@ func (b *backend) fetchByDragonfly(ctx context.Context, target string, cfg *conf
return err
}

logrus.Infof("fetch: successfully fetched layers via dragonfly [count: %d]", len(layers))
logrus.Infof("fetch: fetched %d layers via dragonfly", len(layers))
return nil
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/backend/inspect.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ type InspectedModelArtifactLayer struct {

// Inspect inspects the target from the storage.
func (b *backend) Inspect(ctx context.Context, target string, cfg *config.Inspect) (any, error) {
logrus.Infof("inspect: starting inspect operation for target %s [config: %+v]", target, cfg)
logrus.Infof("inspect: inspecting target %s", target)
_, err := ParseReference(target)
if err != nil {
return nil, fmt.Errorf("failed to parse target: %w", err)
Expand Down Expand Up @@ -129,6 +129,6 @@ func (b *backend) Inspect(ctx context.Context, target string, cfg *config.Inspec
})
}

logrus.Infof("inspect: successfully inspected target %s", target)
logrus.Infof("inspect: inspected target %s", target)
return inspectedModelArtifact, nil
}
4 changes: 2 additions & 2 deletions pkg/backend/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ type ModelArtifact struct {

// List lists all the model artifacts.
func (b *backend) List(ctx context.Context) ([]*ModelArtifact, error) {
logrus.Info("list: starting list operation for model artifacts")
logrus.Infof("list: listing model artifacts")
modelArtifacts := []*ModelArtifact{}

// list all the repositories.
Expand Down Expand Up @@ -79,7 +79,7 @@ func (b *backend) List(ctx context.Context) ([]*ModelArtifact, error) {
return modelArtifacts[i].CreatedAt.After(modelArtifacts[j].CreatedAt)
})

logrus.Infof("list: successfully listed model artifacts [count: %d]", len(modelArtifacts))
logrus.Infof("list: listed %d model artifacts", len(modelArtifacts))
return modelArtifacts, nil
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/backend/login.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (

// Login logs into a registry.
func (b *backend) Login(ctx context.Context, registry, username, password string, cfg *config.Login) error {
logrus.Infof("login: starting login operation for registry %s [user: %s]", registry, username)
logrus.Infof("login: logging in to registry %s [user: %s]", registry, username)
// read credentials from docker store.
store, err := credentials.NewStoreFromDocker(credentials.StoreOptions{AllowPlaintextPut: true})
if err != nil {
Expand Down Expand Up @@ -70,6 +70,6 @@ func (b *backend) Login(ctx context.Context, registry, username, password string
return err
}

logrus.Infof("login: successfully logged into registry %s [user: %s]", registry, username)
logrus.Infof("login: logged in to registry %s [user: %s]", registry, username)
return nil
}
4 changes: 2 additions & 2 deletions pkg/backend/logout.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (

// Logout logs out of a registry.
func (b *backend) Logout(ctx context.Context, registry string) error {
logrus.Infof("logout: starting logout operation for registry %s", registry)
logrus.Infof("logout: logging out of registry %s", registry)
// read credentials from docker store.
store, err := credentials.NewStoreFromDocker(credentials.StoreOptions{AllowPlaintextPut: true})
if err != nil {
Expand All @@ -37,6 +37,6 @@ func (b *backend) Logout(ctx context.Context, registry string) error {
return err
}

logrus.Infof("logout: successfully logged out of registry %s", registry)
logrus.Infof("logout: logged out of registry %s", registry)
return nil
}
6 changes: 3 additions & 3 deletions pkg/backend/processor/base.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ type base struct {

// Process implements the Processor interface, which can be reused by other processors.
func (b *base) Process(ctx context.Context, builder build.Builder, workDir string, opts ...ProcessOption) ([]ocispec.Descriptor, error) {
logrus.Infof("processor: starting %s processing [mediaType: %s, patterns: %v]", b.name, b.mediaType, b.patterns)
logrus.Infof("processor: processing %s files [mediaType: %s, patterns: %v]", b.name, b.mediaType, b.patterns)

processOpts := &processOptions{}
for _, opt := range opts {
Expand Down Expand Up @@ -101,7 +101,7 @@ func (b *base) Process(ctx context.Context, builder build.Builder, workDir strin

sort.Strings(matchedPaths)

logrus.Infof("processor: processing %s files [count: %d]", b.name, len(matchedPaths))
logrus.Infof("processor: matched %s files [count: %d]", b.name, len(matchedPaths))

var (
mu sync.Mutex
Expand Down Expand Up @@ -186,7 +186,7 @@ func (b *base) Process(ctx context.Context, builder build.Builder, workDir strin
return nil, err
}

logrus.Infof("processor: successfully processed %s files [count: %d]", b.name, len(matchedPaths))
logrus.Infof("processor: processed %s files [count: %d]", b.name, len(matchedPaths))

sort.Slice(descriptors, func(i int, j int) bool {
// Sort by filepath by default.
Expand Down
4 changes: 2 additions & 2 deletions pkg/backend/prune.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (

// Prune prunes the unused blobs and clean up the storage.
func (b *backend) Prune(ctx context.Context, dryRun, removeUntagged bool) error {
logrus.Infof("prune: starting prune operation for unused blobs and storage cleanup")
logrus.Infof("prune: pruning unused blobs")

if err := b.store.PerformGC(ctx, dryRun, removeUntagged); err != nil {
return fmt.Errorf("faile to perform gc: %w", err)
Expand All @@ -35,6 +35,6 @@ func (b *backend) Prune(ctx context.Context, dryRun, removeUntagged bool) error
return fmt.Errorf("failed to perform purge uploads: %w", err)
}

logrus.Infof("prune: successfully pruned unused blobs and cleaned up storage")
logrus.Infof("prune: pruned unused blobs")
return nil
}
17 changes: 10 additions & 7 deletions pkg/backend/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ import (

// Pull pulls an artifact from a registry.
func (b *backend) Pull(ctx context.Context, target string, cfg *config.Pull) error {
logrus.Infof("pull: starting pull operation for target %s [config: %+v]", target, cfg)
logrus.Infof("pull: pulling artifact %s", target)

// pullByDragonfly is called if a Dragonfly endpoint is specified in the configuration.
if cfg.DragonflyEndpoint != "" {
logrus.Infof("pull: using dragonfly for target %s", target)
logrus.Infof("pull: using dragonfly for %s", target)
return b.pullByDragonfly(ctx, target, cfg)
}

Expand Down Expand Up @@ -104,7 +104,7 @@ func (b *backend) Pull(ctx context.Context, target string, cfg *config.Pull) err
}
}

logrus.Infof("pull: processing layers for target %s [count: %d]", target, len(manifest.Layers))
logrus.Infof("pull: pulling %d layers for %s", len(manifest.Layers), target)
for _, layer := range manifest.Layers {
g.Go(func() error {
select {
Expand Down Expand Up @@ -134,7 +134,7 @@ func (b *backend) Pull(ctx context.Context, target string, cfg *config.Pull) err
return fmt.Errorf("failed to pull blob to local: %w", err)
}

logrus.Infof("pull: successfully processed layers [count: %d]", len(manifest.Layers))
logrus.Infof("pull: layers pulled [count: %d]", len(manifest.Layers))

// return earlier if extract from remote is enabled as config and manifest
// are not needed for this operation.
Expand Down Expand Up @@ -163,10 +163,10 @@ func (b *backend) Pull(ctx context.Context, target string, cfg *config.Pull) err
if err := exportModelArtifact(ctx, dst, manifest, repo, extractCfg); err != nil {
return fmt.Errorf("failed to export the artifact to the output directory: %w", err)
}
logrus.Infof("pull: successfully pulled and extracted artifact %s", target)
logrus.Infof("pull: pulled and extracted artifact %s", target)
}

logrus.Infof("pull: successfully pulled artifact %s", target)
logrus.Infof("pull: pulled artifact %s", target)
return nil
}

Expand Down Expand Up @@ -258,7 +258,10 @@ func pullAndExtractFromRemote(ctx context.Context, pb *internalpb.ProgressBar, p

if err := extractLayer(desc, outputDir, reader); err != nil {
if errors.Is(err, codec.ErrAlreadyUpToDate) {
logrus.Debugf("pull: skipped extracting blob %s because target is up-to-date", desc.Digest.String())
logrus.Debugf(
"pull: skipping extraction for blob %s, already up-to-date",
desc.Digest.String(),
)
pb.Complete(desc.Digest.String(), fmt.Sprintf("%s %s", internalpb.NormalizePrompt("Skipped blob"), desc.Digest.String()))
return nil
}
Expand Down
Loading