Add OCI internal package

Implement OCI artifacts operations using crane

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

go.mod

@@ -72,8 +72,13 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.11.4 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/cli v20.10.16+incompatible // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/docker v20.10.16+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
 	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
@@ -115,6 +120,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/klauspost/compress v1.15.4 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
@@ -132,6 +138,8 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
@@ -139,10 +147,12 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/russross/blackfriday v1.5.2 // indirect
+	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/texttheater/golang-levenshtein v1.0.1 // indirect
+	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
 	github.com/xanzy/go-gitlab v0.58.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect

internal/oci/build.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// Build archives the given directory as a tarball to the given local path.
+// While archiving, any environment specific data (for example, the user and group name) is stripped from file headers.
+func Build(artifactPath, sourceDir string) error {
+	if f, err := os.Stat(sourceDir); os.IsNotExist(err) || !f.IsDir() {
+		return fmt.Errorf("invalid source dir path: %s", sourceDir)
+	}
+
+	tf, err := os.CreateTemp(filepath.Split(sourceDir))
+	if err != nil {
+		return err
+	}
+	tmpName := tf.Name()
+	defer func() {
+		if err != nil {
+			os.Remove(tmpName)
+		}
+	}()
+
+	sz := &writeCounter{}
+	mw := io.MultiWriter(tf, sz)
+
+	gw := gzip.NewWriter(mw)
+	tw := tar.NewWriter(gw)
+	if err := filepath.Walk(sourceDir, func(p string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Ignore anything that is not a file or directories e.g. symlinks
+		if m := fi.Mode(); !(m.IsRegular() || m.IsDir()) {
+			return nil
+		}
+
+		header, err := tar.FileInfoHeader(fi, p)
+		if err != nil {
+			return err
+		}
+		// The name needs to be modified to maintain directory structure
+		// as tar.FileInfoHeader only has access to the base name of the file.
+		// Ref: https://golang.org/src/archive/tar/common.go?#L626
+		relFilePath := p
+		if filepath.IsAbs(sourceDir) {
+			relFilePath, err = filepath.Rel(sourceDir, p)
+			if err != nil {
+				return err
+			}
+		}
+		header.Name = relFilePath
+
+		// Remove any environment specific data.
+		header.Gid = 0
+		header.Uid = 0
+		header.Uname = ""
+		header.Gname = ""
+		header.ModTime = time.Time{}
+		header.AccessTime = time.Time{}
+		header.ChangeTime = time.Time{}
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		if !fi.Mode().IsRegular() {
+			return nil
+		}
+		f, err := os.Open(p)
+		if err != nil {
+			f.Close()
+			return err
+		}
+		if _, err := io.Copy(tw, f); err != nil {
+			f.Close()
+			return err
+		}
+		return f.Close()
+	}); err != nil {
+		tw.Close()
+		gw.Close()
+		tf.Close()
+		return err
+	}
+
+	if err := tw.Close(); err != nil {
+		gw.Close()
+		tf.Close()
+		return err
+	}
+	if err := gw.Close(); err != nil {
+		tf.Close()
+		return err
+	}
+	if err := tf.Close(); err != nil {
+		return err
+	}
+
+	if err := os.Chmod(tmpName, 0o640); err != nil {
+		return err
+	}
+
+	return os.Rename(tmpName, artifactPath)
+}
+
+type writeCounter struct {
+	written int64
+}
+
+func (wc *writeCounter) Write(p []byte) (int, error) {
+	n := len(p)
+	wc.written += int64(n)
+	return n, nil
+}

internal/oci/meta.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+)
+
+const (
+	SourceAnnotation   = "source.toolkit.fluxcd.io/url"
+	RevisionAnnotation = "source.toolkit.fluxcd.io/revision"
+)
+
+type Metadata struct {
+	Source   string `json:"source_url"`
+	Revision string `json:"source_revision"`
+	Digest   string `json:"digest"`
+}
+
+func (m *Metadata) ToAnnotations() map[string]string {
+	annotations := map[string]string{
+		SourceAnnotation:   m.Source,
+		RevisionAnnotation: m.Revision,
+	}
+
+	return annotations
+}
+
+func GetMetadata(annotations map[string]string) (*Metadata, error) {
+	source, ok := annotations[SourceAnnotation]
+	if !ok {
+		return nil, fmt.Errorf("'%s' annotation not found", SourceAnnotation)
+	}
+
+	revision, ok := annotations[RevisionAnnotation]
+	if !ok {
+		return nil, fmt.Errorf("'%s' annotation not found", RevisionAnnotation)
+	}
+
+	m := Metadata{
+		Source:   source,
+		Revision: revision,
+	}
+
+	return &m, nil
+}

internal/oci/pull.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fluxcd/pkg/untar"
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/name"
+)
+
+// Pull downloads an artifact from an OCI repository and extracts the content to the given directory.
+func Pull(ctx context.Context, url, outDir string) (*Metadata, error) {
+	ref, err := name.ParseReference(url)
+	if err != nil {
+		return nil, fmt.Errorf("invalid URL: %w", err)
+	}
+
+	img, err := crane.Pull(url, craneOptions(ctx)...)
+	if err != nil {
+		return nil, err
+	}
+
+	digest, err := img.Digest()
+	if err != nil {
+		return nil, fmt.Errorf("parsing digest failed: %w", err)
+	}
+
+	manifest, err := img.Manifest()
+	if err != nil {
+		return nil, fmt.Errorf("parsing manifest failed: %w", err)
+	}
+
+	meta, err := GetMetadata(manifest.Annotations)
+	if err != nil {
+		return nil, err
+	}
+	meta.Digest = ref.Context().Digest(digest.String()).String()
+
+	layers, err := img.Layers()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list layers: %w", err)
+	}
+
+	if len(layers) < 1 {
+		return nil, fmt.Errorf("no layers found in artifact")
+	}
+
+	blob, err := layers[0].Compressed()
+	if err != nil {
+		return nil, fmt.Errorf("extracting first layer failed: %w", err)
+	}
+
+	if _, err = untar.Untar(blob, outDir); err != nil {
+		return nil, fmt.Errorf("failed to untar first layer: %w", err)
+	}
+
+	return meta, nil
+}

internal/oci/push.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/name"
+	gcrv1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+)
+
+// Push creates an artifact from the given directory, uploads the artifact
+// to the given OCI repository and returns the digest.
+func Push(ctx context.Context, url, sourceDir string, meta Metadata) (string, error) {
+	ref, err := name.ParseReference(url)
+	if err != nil {
+		return "", fmt.Errorf("invalid URL: %w", err)
+	}
+
+	tmpDir, err := os.MkdirTemp("", "oci")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	tmpFile := filepath.Join(tmpDir, "artifact.tgz")
+
+	if err := Build(tmpFile, sourceDir); err != nil {
+		return "", err
+	}
+
+	img, err := crane.Append(empty.Image, tmpFile)
+	if err != nil {
+		return "", fmt.Errorf("appeding content to artifact failed: %w", err)
+	}
+
+	img = mutate.Annotations(img, meta.ToAnnotations()).(gcrv1.Image)
+
+	if err := crane.Push(img, url, craneOptions(ctx)...); err != nil {
+		return "", fmt.Errorf("pushing artifact failed: %w", err)
+	}
+
+	digest, err := img.Digest()
+	if err != nil {
+		return "", fmt.Errorf("parsing artifact digest failed: %w", err)
+	}
+
+	return ref.Context().Digest(digest.String()).String(), err
+}
+
+func craneOptions(ctx context.Context) []crane.Option {
+	return []crane.Option{
+		crane.WithContext(ctx),
+		crane.WithUserAgent("flux/v2"),
+		crane.WithPlatform(&gcrv1.Platform{
+			Architecture: "flux",
+			OS:           "flux",
+			OSVersion:    "v2",
+		}),
+	}
+}

internal/oci/tag.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"context"
+	"fmt"
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/name"
+)
+
+// Tag creates a new tag for the given artifact using the same OCI repository as the origin.
+func Tag(ctx context.Context, url, tag string) (string, error) {
+	ref, err := name.ParseReference(url)
+	if err != nil {
+		return "", fmt.Errorf("invalid URL: %w", err)
+	}
+
+	if err := crane.Tag(url, tag, craneOptions(ctx)...); err != nil {
+		return "", err
+	}
+
+	dst := ref.Context().Tag(tag)
+
+	return dst.Name(), nil
+}