Mirrors/flux2
1
0
Fork 0
mirror of synced 2026-02-06 19:05:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add OCI internal package

Browse Source 
Implement OCI artifacts operations using crane

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
This commit is contained in:
Stefan Prodan
2022-06-21 12:33:50 +03:00
parent ac50aea21f
commit e927d39a27
7 changed files with 829 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
go.mod
View File

@@ -72,8 +72,13 @@ require (
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/containerd/stargz-snapshotter/estargz v0.11.4 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.16+incompatible // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker v20.10.16+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
github.com/emicklei/go-restful v2.9.5+incompatible // indirect
github.com/emirpasic/gods v1.12.0 // indirect
@@ -115,6 +120,7 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
github.com/klauspost/compress v1.15.4 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
@@ -132,6 +138,8 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
@@ -139,10 +147,12 @@ require (
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday v1.5.2 // indirect
github.com/russross/blackfriday v1.6.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
github.com/xanzy/go-gitlab v0.58.0 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect

426
go.sum
View File

File diff suppressed because it is too large Load Diff

139
internal/oci/build.go Normal file
View File

@@ -0,0 +1,139 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oci
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"os"
"path/filepath"
"time"
)
// Build archives the given directory as a tarball to the given local path.
// While archiving, any environment specific data (for example, the user and group name) is stripped from file headers.
func Build(artifactPath, sourceDir string) error {
if f, err := os.Stat(sourceDir); os.IsNotExist(err) || !f.IsDir() {
return fmt.Errorf("invalid source dir path: %s", sourceDir)
}
tf, err := os.CreateTemp(filepath.Split(sourceDir))
if err != nil {
return err
}
tmpName := tf.Name()
defer func() {
if err != nil {
os.Remove(tmpName)
}
}()
sz := &writeCounter{}
mw := io.MultiWriter(tf, sz)
gw := gzip.NewWriter(mw)
tw := tar.NewWriter(gw)
if err := filepath.Walk(sourceDir, func(p string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
// Ignore anything that is not a file or directories e.g. symlinks
if m := fi.Mode(); !(m.IsRegular() || m.IsDir()) {
return nil
}
header, err := tar.FileInfoHeader(fi, p)
if err != nil {
return err
}
// The name needs to be modified to maintain directory structure
// as tar.FileInfoHeader only has access to the base name of the file.
// Ref: https://golang.org/src/archive/tar/common.go?#L626
relFilePath := p
if filepath.IsAbs(sourceDir) {
relFilePath, err = filepath.Rel(sourceDir, p)
if err != nil {
return err
}
}
header.Name = relFilePath
// Remove any environment specific data.
header.Gid = 0
header.Uid = 0
header.Uname = ""
header.Gname = ""
header.ModTime = time.Time{}
header.AccessTime = time.Time{}
header.ChangeTime = time.Time{}
if err := tw.WriteHeader(header); err != nil {
return err
}
if !fi.Mode().IsRegular() {
return nil
}
f, err := os.Open(p)
if err != nil {
f.Close()
return err
}
if _, err := io.Copy(tw, f); err != nil {
f.Close()
return err
}
return f.Close()
}); err != nil {
tw.Close()
gw.Close()
tf.Close()
return err
}
if err := tw.Close(); err != nil {
gw.Close()
tf.Close()
return err
}
if err := gw.Close(); err != nil {
tf.Close()
return err
}
if err := tf.Close(); err != nil {
return err
}
if err := os.Chmod(tmpName, 0o640); err != nil {
return err
}
return os.Rename(tmpName, artifactPath)
}
type writeCounter struct {
written int64
}
func (wc *writeCounter) Write(p []byte) (int, error) {
n := len(p)
wc.written += int64(n)
return n, nil
}

60
internal/oci/meta.go Normal file
View File

@@ -0,0 +1,60 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oci
import (
"fmt"
)
const (
SourceAnnotation = "source.toolkit.fluxcd.io/url"
RevisionAnnotation = "source.toolkit.fluxcd.io/revision"
)
type Metadata struct {
Source string `json:"source_url"`
Revision string `json:"source_revision"`
Digest string `json:"digest"`
}
func (m *Metadata) ToAnnotations() map[string]string {
annotations := map[string]string{
SourceAnnotation: m.Source,
RevisionAnnotation: m.Revision,
}
return annotations
}
func GetMetadata(annotations map[string]string) (*Metadata, error) {
source, ok := annotations[SourceAnnotation]
if !ok {
return nil, fmt.Errorf("'%s' annotation not found", SourceAnnotation)
}
revision, ok := annotations[RevisionAnnotation]
if !ok {
return nil, fmt.Errorf("'%s' annotation not found", RevisionAnnotation)
}
m := Metadata{
Source: source,
Revision: revision,
}
return &m, nil
}

75
internal/oci/pull.go Normal file
View File

@@ -0,0 +1,75 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oci
import (
"context"
"fmt"
"github.com/fluxcd/pkg/untar"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
)
// Pull downloads an artifact from an OCI repository and extracts the content to the given directory.
func Pull(ctx context.Context, url, outDir string) (*Metadata, error) {
ref, err := name.ParseReference(url)
if err != nil {
return nil, fmt.Errorf("invalid URL: %w", err)
}
img, err := crane.Pull(url, craneOptions(ctx)...)
if err != nil {
return nil, err
}
digest, err := img.Digest()
if err != nil {
return nil, fmt.Errorf("parsing digest failed: %w", err)
}
manifest, err := img.Manifest()
if err != nil {
return nil, fmt.Errorf("parsing manifest failed: %w", err)
}
meta, err := GetMetadata(manifest.Annotations)
if err != nil {
return nil, err
}
meta.Digest = ref.Context().Digest(digest.String()).String()
layers, err := img.Layers()
if err != nil {
return nil, fmt.Errorf("failed to list layers: %w", err)
}
if len(layers) < 1 {
return nil, fmt.Errorf("no layers found in artifact")
}
blob, err := layers[0].Compressed()
if err != nil {
return nil, fmt.Errorf("extracting first layer failed: %w", err)
}
if _, err = untar.Untar(blob, outDir); err != nil {
return nil, fmt.Errorf("failed to untar first layer: %w", err)
}
return meta, nil
}

81
internal/oci/push.go Normal file
View File

@@ -0,0 +1,81 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oci
import (
"context"
"fmt"
"os"
"path/filepath"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
gcrv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
)
// Push creates an artifact from the given directory, uploads the artifact
// to the given OCI repository and returns the digest.
func Push(ctx context.Context, url, sourceDir string, meta Metadata) (string, error) {
ref, err := name.ParseReference(url)
if err != nil {
return "", fmt.Errorf("invalid URL: %w", err)
}
tmpDir, err := os.MkdirTemp("", "oci")
if err != nil {
return "", err
}
defer os.RemoveAll(tmpDir)
tmpFile := filepath.Join(tmpDir, "artifact.tgz")
if err := Build(tmpFile, sourceDir); err != nil {
return "", err
}
img, err := crane.Append(empty.Image, tmpFile)
if err != nil {
return "", fmt.Errorf("appeding content to artifact failed: %w", err)
}
img = mutate.Annotations(img, meta.ToAnnotations()).(gcrv1.Image)
if err := crane.Push(img, url, craneOptions(ctx)...); err != nil {
return "", fmt.Errorf("pushing artifact failed: %w", err)
}
digest, err := img.Digest()
if err != nil {
return "", fmt.Errorf("parsing artifact digest failed: %w", err)
}
return ref.Context().Digest(digest.String()).String(), err
}
func craneOptions(ctx context.Context) []crane.Option {
return []crane.Option{
crane.WithContext(ctx),
crane.WithUserAgent("flux/v2"),
crane.WithPlatform(&gcrv1.Platform{
Architecture: "flux",
OS: "flux",
OSVersion: "v2",
}),
}
}

40
internal/oci/tag.go Normal file
View File

@@ -0,0 +1,40 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oci
import (
"context"
"fmt"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
)
// Tag creates a new tag for the given artifact using the same OCI repository as the origin.
func Tag(ctx context.Context, url, tag string) (string, error) {
ref, err := name.ParseReference(url)
if err != nil {
return "", fmt.Errorf("invalid URL: %w", err)
}
if err := crane.Tag(url, tag, craneOptions(ctx)...); err != nil {
return "", err
}
dst := ref.Context().Tag(tag)
return dst.Name(), nil
}