mirror of https://github.com/fluxcd/flux2.git
Add OCI internal package
Implement OCI artifacts operations using crane Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>pull/2971/head
Stefan Prodan
3 years ago
parent
ac50aea21f
commit
e927d39a27
@ -0,0 +1,139 @@
|
||||
/*
|
||||
Copyright 2022 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Build archives the given directory as a tarball to the given local path.
|
||||
// While archiving, any environment specific data (for example, the user and group name) is stripped from file headers.
|
||||
func Build(artifactPath, sourceDir string) error {
|
||||
if f, err := os.Stat(sourceDir); os.IsNotExist(err) || !f.IsDir() {
|
||||
return fmt.Errorf("invalid source dir path: %s", sourceDir)
|
||||
}
|
||||
|
||||
tf, err := os.CreateTemp(filepath.Split(sourceDir))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tf.Name()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
os.Remove(tmpName)
|
||||
}
|
||||
}()
|
||||
|
||||
sz := &writeCounter{}
|
||||
mw := io.MultiWriter(tf, sz)
|
||||
|
||||
gw := gzip.NewWriter(mw)
|
||||
tw := tar.NewWriter(gw)
|
||||
if err := filepath.Walk(sourceDir, func(p string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Ignore anything that is not a file or directories e.g. symlinks
|
||||
if m := fi.Mode(); !(m.IsRegular() || m.IsDir()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(fi, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The name needs to be modified to maintain directory structure
|
||||
// as tar.FileInfoHeader only has access to the base name of the file.
|
||||
// Ref: https://golang.org/src/archive/tar/common.go?#L626
|
||||
relFilePath := p
|
||||
if filepath.IsAbs(sourceDir) {
|
||||
relFilePath, err = filepath.Rel(sourceDir, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
header.Name = relFilePath
|
||||
|
||||
// Remove any environment specific data.
|
||||
header.Gid = 0
|
||||
header.Uid = 0
|
||||
header.Uname = ""
|
||||
header.Gname = ""
|
||||
header.ModTime = time.Time{}
|
||||
header.AccessTime = time.Time{}
|
||||
header.ChangeTime = time.Time{}
|
||||
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !fi.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
f, err := os.Open(p)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(tw, f); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}); err != nil {
|
||||
tw.Close()
|
||||
gw.Close()
|
||||
tf.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tw.Close(); err != nil {
|
||||
gw.Close()
|
||||
tf.Close()
|
||||
return err
|
||||
}
|
||||
if err := gw.Close(); err != nil {
|
||||
tf.Close()
|
||||
return err
|
||||
}
|
||||
if err := tf.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.Chmod(tmpName, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Rename(tmpName, artifactPath)
|
||||
}
|
||||
|
||||
type writeCounter struct {
|
||||
written int64
|
||||
}
|
||||
|
||||
func (wc *writeCounter) Write(p []byte) (int, error) {
|
||||
n := len(p)
|
||||
wc.written += int64(n)
|
||||
return n, nil
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
/*
|
||||
Copyright 2022 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
SourceAnnotation = "source.toolkit.fluxcd.io/url"
|
||||
RevisionAnnotation = "source.toolkit.fluxcd.io/revision"
|
||||
)
|
||||
|
||||
type Metadata struct {
|
||||
Source string `json:"source_url"`
|
||||
Revision string `json:"source_revision"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
func (m *Metadata) ToAnnotations() map[string]string {
|
||||
annotations := map[string]string{
|
||||
SourceAnnotation: m.Source,
|
||||
RevisionAnnotation: m.Revision,
|
||||
}
|
||||
|
||||
return annotations
|
||||
}
|
||||
|
||||
func GetMetadata(annotations map[string]string) (*Metadata, error) {
|
||||
source, ok := annotations[SourceAnnotation]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("'%s' annotation not found", SourceAnnotation)
|
||||
}
|
||||
|
||||
revision, ok := annotations[RevisionAnnotation]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("'%s' annotation not found", RevisionAnnotation)
|
||||
}
|
||||
|
||||
m := Metadata{
|
||||
Source: source,
|
||||
Revision: revision,
|
||||
}
|
||||
|
||||
return &m, nil
|
||||
}
|
||||
@ -0,0 +1,75 @@
|
||||
/*
|
||||
Copyright 2022 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/fluxcd/pkg/untar"
|
||||
"github.com/google/go-containerregistry/pkg/crane"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
)
|
||||
|
||||
// Pull downloads an artifact from an OCI repository and extracts the content to the given directory.
|
||||
func Pull(ctx context.Context, url, outDir string) (*Metadata, error) {
|
||||
ref, err := name.ParseReference(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
|
||||
img, err := crane.Pull(url, craneOptions(ctx)...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
digest, err := img.Digest()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing digest failed: %w", err)
|
||||
}
|
||||
|
||||
manifest, err := img.Manifest()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing manifest failed: %w", err)
|
||||
}
|
||||
|
||||
meta, err := GetMetadata(manifest.Annotations)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta.Digest = ref.Context().Digest(digest.String()).String()
|
||||
|
||||
layers, err := img.Layers()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list layers: %w", err)
|
||||
}
|
||||
|
||||
if len(layers) < 1 {
|
||||
return nil, fmt.Errorf("no layers found in artifact")
|
||||
}
|
||||
|
||||
blob, err := layers[0].Compressed()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extracting first layer failed: %w", err)
|
||||
}
|
||||
|
||||
if _, err = untar.Untar(blob, outDir); err != nil {
|
||||
return nil, fmt.Errorf("failed to untar first layer: %w", err)
|
||||
}
|
||||
|
||||
return meta, nil
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
/*
|
||||
Copyright 2022 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/google/go-containerregistry/pkg/crane"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
gcrv1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/empty"
|
||||
"github.com/google/go-containerregistry/pkg/v1/mutate"
|
||||
)
|
||||
|
||||
// Push creates an artifact from the given directory, uploads the artifact
|
||||
// to the given OCI repository and returns the digest.
|
||||
func Push(ctx context.Context, url, sourceDir string, meta Metadata) (string, error) {
|
||||
ref, err := name.ParseReference(url)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "oci")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
tmpFile := filepath.Join(tmpDir, "artifact.tgz")
|
||||
|
||||
if err := Build(tmpFile, sourceDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
img, err := crane.Append(empty.Image, tmpFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("appeding content to artifact failed: %w", err)
|
||||
}
|
||||
|
||||
img = mutate.Annotations(img, meta.ToAnnotations()).(gcrv1.Image)
|
||||
|
||||
if err := crane.Push(img, url, craneOptions(ctx)...); err != nil {
|
||||
return "", fmt.Errorf("pushing artifact failed: %w", err)
|
||||
}
|
||||
|
||||
digest, err := img.Digest()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parsing artifact digest failed: %w", err)
|
||||
}
|
||||
|
||||
return ref.Context().Digest(digest.String()).String(), err
|
||||
}
|
||||
|
||||
func craneOptions(ctx context.Context) []crane.Option {
|
||||
return []crane.Option{
|
||||
crane.WithContext(ctx),
|
||||
crane.WithUserAgent("flux/v2"),
|
||||
crane.WithPlatform(&gcrv1.Platform{
|
||||
Architecture: "flux",
|
||||
OS: "flux",
|
||||
OSVersion: "v2",
|
||||
}),
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
/*
|
||||
Copyright 2022 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package oci
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-containerregistry/pkg/crane"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
)
|
||||
|
||||
// Tag creates a new tag for the given artifact using the same OCI repository as the origin.
|
||||
func Tag(ctx context.Context, url, tag string) (string, error) {
|
||||
ref, err := name.ParseReference(url)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
|
||||
if err := crane.Tag(url, tag, craneOptions(ctx)...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
dst := ref.Context().Tag(tag)
|
||||
|
||||
return dst.Name(), nil
|
||||
}
|
||||
Loading…
Reference in New Issue