Skip to content

Commit b1bb97f

Browse files
flouthocflouthoc
committed
buildkit: supports additionalBuildContext in builds via --build-context
As builds got more complicated, the ability to only access files from one location became quite limiting. With `multi-stage` builds where you can `copy` files from other parts of the Containerfile by adding the `--from` flag and pointing it to the name of another Containerfile stage or a remote image. The new named build context feature is an extension of this pattern. You can now define additional build contexts when running the build command, give them a name, and then access them inside a Dockerfile the same way you previously did with build stages. Additional build contexts can be defined with a new `--build-context [name]=[value]` flag. The key component defines the name for your build context and the value can be: ```console Local directory – e.g. --build-context project2=../path/to/project2/src HTTP URL to a tarball – e.g. --build-context src=https://example.org/releases/src.tar Container image – Define with a docker-image:// prefix, e.g. --build-context alpine=docker-image://alpine:3.15, ( also supports docker://, container-image:// ) ``` On the Containerfile side, you can reference the build context on all commands that accept the “from” parameter. Here’s how that might look: ```Dockerfile FROM [name] COPY --from=[name] ... RUN --mount=from=[name] … ``` The value of [name] is matched with the following priority order: * Named build context defined with `--build-context [name]=..` * Stage defined with `AS [name]` inside Dockerfile * Remote image `[name]` in a container registry Added Features * Pinning images for `FROM` and `COPY` * Specifying multiple buildcontexts from different projects and using them with `--from` in `ADD` and `COPY` directive * Override a Remote Dependency with a Local One. * Using additional context from external `Tar` Signed-off-by: Aditya R <[email protected]>
1 parent c1a3cd1 commit b1bb97f

File tree

12 files changed

+279
-24
lines changed

12 files changed

+279
-24
lines changed

‎cmd/buildah/build.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,16 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error {
157157
}
158158
}
159159

160+
additionalBuildContext := make(map[string]define.AdditionalBuildContext)
161+
if c.Flag("build-context").Changed {
162+
for _, contextString := range iopts.BuildContext {
163+
av := strings.SplitN(contextString, "=", 2)
164+
if len(av) > 1 {
165+
additionalBuildContext[av[0]] = parse.GetAdditionalBuildContext(av[1])
166+
}
167+
}
168+
}
169+
160170
containerfiles := getContainerfiles(iopts.File)
161171
format, err := getFormat(iopts.Format)
162172
if err != nil {
@@ -344,6 +354,7 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error {
344354
Annotations: iopts.Annotation,
345355
Architecture: systemContext.ArchitectureChoice,
346356
Args: args,
357+
BuildContext: additionalBuildContext,
347358
BlobDirectory: iopts.BlobCache,
348359
CNIConfigDir: iopts.CNIConfigDir,
349360
CNIPluginPath: iopts.CNIPlugInPath,

‎define/build.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,16 @@ import (
1111
"golang.org/x/sync/semaphore"
1212
)
1313

14+
// AdditionalBuildContext contains verbose details about a parsed build context from --build-context
15+
type AdditionalBuildContext struct {
16+
// AdditionalBuildContext is a URL to externla tar.
17+
IsURL bool
18+
// AdditionalBuildContext is an image.
19+
IsImage bool
20+
// Actual source value.
21+
Value string
22+
}
23+
1424
// CommonBuildOptions are resources that can be defined by flags for both buildah from and build
1525
type CommonBuildOptions struct {
1626
// AddHost is the list of hostnames to add to the build container's /etc/hosts.
@@ -121,6 +131,8 @@ type BuildOptions struct {
121131
Compression archive.Compression
122132
// Arguments which can be interpolated into Dockerfiles
123133
Args map[string]string
134+
// Map of external additional build contexts
135+
BuildContext map[string]AdditionalBuildContext
124136
// Name of the image to write to.
125137
Output string
126138
// BuildOutput specifies if any custom build output is selected for following build.

‎docs/buildah-build.1.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,32 @@ resulting image's configuration.
6868
Please refer to the [BUILD TIME VARIABLES](#build-time-variables) section for the
6969
list of variables that can be overridden within the Containerfile at run time.
7070

71+
**--build-context** *name=value*
72+
73+
Specify additional buildah context using its short name and its value. Additional
74+
build context can be used in the same manner as we access different stages in `COPY`
75+
and `ADD` directive.
76+
77+
Valid values could be:
78+
* Local directory – e.g. --build-context project2=../path/to/project2/src
79+
* HTTP URL to a tarball – e.g. --build-context src=https://example.org/releases/src.tar
80+
* Container image – Define with a docker-image:// prefix, e.g. --build-context alpine=docker-image://alpine:3.15, ( also supports docker://, container-image://)
81+
82+
On the Containerfile side, you can reference the build context on all commands that accept the “from” parameter.
83+
Here’s how that might look:
84+
85+
```Dockerfile
86+
FROM [name]
87+
COPY --from=[name] ...
88+
RUN --mount=from=[name] …
89+
```
90+
91+
The value of `[name]` is matched with the following priority order:
92+
93+
* Named build context defined with --build-context [name]=..
94+
* Stage defined with AS [name] inside Dockerfile
95+
* Remote image [name] in a container registry
96+
7197
**--cache-from**
7298

7399
Images to utilise as potential cache sources. Buildah does not currently support --cache-from so this is a NOOP.

‎imagebuildah/build.go‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B
211211
}
212212

213213
if options.AllPlatforms {
214-
options.Platforms, err = platformsForBaseImages(ctx, logger, paths, files, options.From, options.Args, options.SystemContext)
214+
options.Platforms, err = platformsForBaseImages(ctx, logger, paths, files, options.From, options.Args, options.BuildContext, options.SystemContext)
215215
if err != nil {
216216
return "", nil, err
217217
}
@@ -502,8 +502,8 @@ func preprocessContainerfileContents(logger *logrus.Logger, containerfile string
502502
// platformsForBaseImages resolves the names of base images from the
503503
// dockerfiles, and if they are all valid references to manifest lists, returns
504504
// the list of platforms that are supported by all of the base images.
505-
func platformsForBaseImages(ctx context.Context, logger *logrus.Logger, dockerfilepaths []string, dockerfiles [][]byte, from string, args map[string]string, systemContext *types.SystemContext) ([]struct{ OS, Arch, Variant string }, error) {
506-
baseImages, err := baseImages(dockerfilepaths, dockerfiles, from, args)
505+
func platformsForBaseImages(ctx context.Context, logger *logrus.Logger, dockerfilepaths []string, dockerfiles [][]byte, from string, args map[string]string, additionalBuildContext map[string]define.AdditionalBuildContext, systemContext *types.SystemContext) ([]struct{ OS, Arch, Variant string }, error) {
506+
baseImages, err := baseImages(dockerfilepaths, dockerfiles, from, args, additionalBuildContext)
507507
if err != nil {
508508
return nil, errors.Wrapf(err, "determining list of base images")
509509
}
@@ -631,7 +631,7 @@ func platformsForBaseImages(ctx context.Context, logger *logrus.Logger, dockerfi
631631
// stage's base image with FROM, and returns the list of base images as
632632
// provided. Each entry in the dockerfilenames slice corresponds to a slice in
633633
// dockerfilecontents.
634-
func baseImages(dockerfilenames []string, dockerfilecontents [][]byte, from string, args map[string]string) ([]string, error) {
634+
func baseImages(dockerfilenames []string, dockerfilecontents [][]byte, from string, args map[string]string, additionalBuildContext map[string]define.AdditionalBuildContext) ([]string, error) {
635635
mainNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(dockerfilecontents[0]))
636636
if err != nil {
637637
return nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfilenames[0])
@@ -670,6 +670,11 @@ func baseImages(dockerfilenames []string, dockerfilecontents [][]byte, from stri
670670
child.Next.Value = from
671671
from = ""
672672
}
673+
if replaceBuildContext, ok := additionalBuildContext[child.Next.Value]; ok {
674+
if replaceBuildContext.IsImage {
675+
child.Next.Value = replaceBuildContext.Value
676+
}
677+
}
673678
base := child.Next.Value
674679
if base != "scratch" && !nicknames[base] {
675680
// TODO: this didn't undergo variable and arg

‎imagebuildah/executor.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ type Executor struct {
126126
imageInfoLock sync.Mutex
127127
imageInfoCache map[string]imageTypeAndHistoryAndDiffIDs
128128
fromOverride string
129+
buildContext map[string]define.AdditionalBuildContext
129130
manifest string
130131
secrets map[string]define.Secret
131132
sshsources map[string]*sshagent.Source
@@ -275,6 +276,7 @@ func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, o
275276
rusageLogFile: rusageLogFile,
276277
imageInfoCache: make(map[string]imageTypeAndHistoryAndDiffIDs),
277278
fromOverride: options.From,
279+
buildContext: options.BuildContext,
278280
manifest: options.Manifest,
279281
secrets: secrets,
280282
sshsources: sshsources,
@@ -607,6 +609,11 @@ func (b *Executor) Build(ctx context.Context, stages imagebuilder.Stages) (image
607609
child.Next.Value = b.fromOverride
608610
b.fromOverride = ""
609611
}
612+
if replaceBuildContext, ok := b.buildContext[child.Next.Value]; ok {
613+
if replaceBuildContext.IsImage {
614+
child.Next.Value = replaceBuildContext.Value
615+
}
616+
}
610617
base := child.Next.Value
611618
if base != "scratch" {
612619
userArgs := argsMapToSlice(stage.Builder.Args)
@@ -626,6 +633,11 @@ func (b *Executor) Build(ctx context.Context, stages imagebuilder.Stages) (image
626633
// was named using argument values, we might
627634
// not record the right value here.
628635
rootfs := strings.TrimPrefix(flag, "--from=")
636+
if replaceBuildContext, ok := b.buildContext[rootfs]; ok {
637+
if replaceBuildContext.IsImage {
638+
rootfs = replaceBuildContext.Value
639+
}
640+
}
629641
b.rootfsMap[rootfs] = true
630642
logrus.Debugf("rootfs needed for COPY in stage %d: %q", stageIndex, rootfs)
631643
}

‎imagebuildah/stage_executor.go‎

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -362,24 +362,48 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err
362362
stripSetgid := false
363363
preserveOwnership := false
364364
contextDir := s.executor.contextDir
365+
365366
if len(copy.From) > 0 {
367+
// If additionalContext is selected do not process anything else
368+
didSelectAdditionalContext := false
366369
// If from has an argument within it, resolve it to its
367370
// value. Otherwise just return the value found.
368371
from, fromErr := imagebuilder.ProcessWord(copy.From, s.stage.Builder.Arguments())
369372
if fromErr != nil {
370373
return errors.Wrapf(fromErr, "unable to resolve argument %q", copy.From)
371374
}
372-
if isStage, err := s.executor.waitForStage(s.ctx, from, s.stages[:s.index]); isStage && err != nil {
373-
return err
375+
if additionalBuildContext, ok := s.executor.buildContext[from]; ok {
376+
if !additionalBuildContext.IsImage {
377+
contextDir = additionalBuildContext.Value
378+
if additionalBuildContext.IsURL {
379+
// additional context contains a tar file
380+
// so download and explode tar to buildah
381+
// temp and point context to that.
382+
untarPath, err := internalUtil.ExportTarFromURL(additionalBuildContext.Value, from)
383+
if err != nil {
384+
return errors.Wrapf(err, "unable to extract Tar from external source %q", additionalBuildContext.Value)
385+
}
386+
// point contextDir to untar
387+
contextDir = untarPath
388+
}
389+
didSelectAdditionalContext = true
390+
} else {
391+
copy.From = additionalBuildContext.Value
392+
}
374393
}
375-
if other, ok := s.executor.stages[from]; ok && other.index < s.index {
376-
contextDir = other.mountPoint
377-
idMappingOptions = &other.builder.IDMappingOptions
378-
} else if builder, ok := s.executor.containerMap[copy.From]; ok {
379-
contextDir = builder.MountPoint
380-
idMappingOptions = &builder.IDMappingOptions
381-
} else {
382-
return errors.Errorf("the stage %q has not been built", copy.From)
394+
if !didSelectAdditionalContext {
395+
if isStage, err := s.executor.waitForStage(s.ctx, from, s.stages[:s.index]); isStage && err != nil {
396+
return err
397+
}
398+
if other, ok := s.executor.stages[from]; ok && other.index < s.index {
399+
contextDir = other.mountPoint
400+
idMappingOptions = &other.builder.IDMappingOptions
401+
} else if builder, ok := s.executor.containerMap[copy.From]; ok {
402+
contextDir = builder.MountPoint
403+
idMappingOptions = &builder.IDMappingOptions
404+
} else {
405+
return errors.Errorf("the stage %q has not been built", copy.From)
406+
}
383407
}
384408
preserveOwnership = true
385409
copyExcludes = excludes
@@ -446,6 +470,15 @@ func (s *StageExecutor) runStageMountPoints(mountList []string) (map[string]inte
446470
if fromErr != nil {
447471
return nil, errors.Wrapf(fromErr, "unable to resolve argument %q", kv[1])
448472
}
473+
// If additional buildContext contains this
474+
// give priority to that and break if additional
475+
// is not an external image.
476+
477+
if additionalBuildContext, ok := s.executor.buildContext[from]; ok {
478+
if additionalBuildContext.IsImage {
479+
from = additionalBuildContext.Value
480+
}
481+
}
449482
// If the source's name corresponds to the
450483
// result of an earlier stage, wait for that
451484
// stage to finish being built.
@@ -923,6 +956,22 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
923956
if fromErr != nil {
924957
return "", nil, errors.Wrapf(fromErr, "unable to resolve argument %q", arr[1])
925958
}
959+
// If additional buildContext contains this
960+
// give priority to that and break if additional
961+
// is not an external image.
962+
963+
if additionalBuildContext, ok := s.executor.buildContext[from]; ok {
964+
if !additionalBuildContext.IsImage {
965+
// We don't need to pull this
966+
// since this additional context
967+
// is not an image.
968+
break
969+
} else {
970+
// replace with image set in build context
971+
from = additionalBuildContext.Value
972+
}
973+
}
974+
926975
// If the source's name corresponds to the
927976
// result of an earlier stage, wait for that
928977
// stage to finish being built.

‎internal/parse/parse.go‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ func GetCacheMount(args []string, store storage.Store, imageMountLabel string, a
309309
// add subdirectory if specified
310310

311311
// cache parent directory
312-
cacheParent := filepath.Join(getTempDir(), BuildahCacheDir)
312+
cacheParent := filepath.Join(internalUtil.GetTempDir(), BuildahCacheDir)
313313
// create cache on host if not present
314314
err = os.MkdirAll(cacheParent, os.FileMode(0755))
315315
if err != nil {
@@ -597,12 +597,3 @@ func GetTmpfsMount(args []string) (specs.Mount, error) {
597597

598598
return newMount, nil
599599
}
600-
601-
/* This is internal function and could be changed at any time */
602-
/* for external usage please refer to buildah/pkg/parse.GetTempDir() */
603-
func getTempDir() string {
604-
if tmpdir, ok := os.LookupEnv("TMPDIR"); ok {
605-
return tmpdir
606-
}
607-
return "/var/tmp"
608-
}

‎internal/types.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
package internal
22

3+
const (
4+
// Temp directory which stores external artifacts which are download for a build.
5+
// Example: tar files from external sources.
6+
BuildahExternalArtifactsDir = "buildah-external-artifacts"
7+
)
8+
39
// Types is internal packages are suspected to change with releases avoid using these outside of buildah
410

511
// StageMountDetails holds the Stage/Image mountpoint returned by StageExecutor

‎internal/util/util.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ import (
66
"path/filepath"
77

88
"github.com/containers/buildah/define"
9+
"github.com/containers/buildah/internal"
910
"github.com/containers/common/libimage"
11+
"github.com/containers/common/pkg/download"
1012
"github.com/containers/image/v5/types"
1113
"github.com/containers/storage"
1214
"github.com/containers/storage/pkg/archive"
@@ -32,6 +34,44 @@ func LookupImage(ctx *types.SystemContext, store storage.Store, image string) (*
3234
return localImage, nil
3335
}
3436

37+
// ExternalTarFromURL takes url of tar and untars into buildah's artifacts dir and returns path.
38+
func ExportTarFromURL(url string, dest string) (string, error) {
39+
// cache parent directory
40+
artifactsParent := filepath.Join(GetTempDir(), internal.BuildahExternalArtifactsDir)
41+
err := os.MkdirAll(artifactsParent, os.FileMode(0755))
42+
if err != nil {
43+
return "", errors.Wrapf(err, "failed while creating the destination path %q", artifactsParent)
44+
}
45+
file, err := download.FromURL(artifactsParent, url)
46+
if err != nil {
47+
return "", err
48+
}
49+
defer os.Remove(file)
50+
unTarPath := filepath.Join(artifactsParent, dest)
51+
err = os.MkdirAll(unTarPath, os.FileMode(0755))
52+
if err != nil {
53+
return "", errors.Wrapf(err, "failed while creating the destination untar path %q", unTarPath)
54+
}
55+
inputFile, err := os.Open(file)
56+
if err != nil {
57+
return "", err
58+
}
59+
defer inputFile.Close()
60+
err = chrootarchive.Untar(inputFile, unTarPath, &archive.TarOptions{})
61+
if err != nil {
62+
return "", err
63+
}
64+
return unTarPath, nil
65+
}
66+
67+
// GetTempDir returns base for a temporary directory on host.
68+
func GetTempDir() string {
69+
if tmpdir, ok := os.LookupEnv("TMPDIR"); ok {
70+
return tmpdir
71+
}
72+
return "/var/tmp"
73+
}
74+
3575
// ExportFromReader reads bytes from given reader and exports to external tar, directory or stdout.
3676
func ExportFromReader(input io.Reader, opts define.BuildOutputOption) error {
3777
var err error

‎pkg/cli/common.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ type BudResults struct {
5353
Annotation []string
5454
Authfile string
5555
BuildArg []string
56+
BuildContext []string
5657
CacheFrom string
5758
CertDir string
5859
Compress bool
@@ -191,6 +192,7 @@ func GetBudFlags(flags *BudResults) pflag.FlagSet {
191192
fs.StringArrayVar(&flags.Annotation, "annotation", []string{}, "set metadata for an image (default [])")
192193
fs.StringVar(&flags.Authfile, "authfile", "", "path of the authentication file.")
193194
fs.StringArrayVar(&flags.BuildArg, "build-arg", []string{}, "`argument=value` to supply to the builder")
195+
fs.StringArrayVar(&flags.BuildContext, "build-context", []string{}, "`argument=value` to supply additional build context to the builder")
194196
fs.StringVar(&flags.CacheFrom, "cache-from", "", "images to utilise as potential cache sources. The build process does not currently support caching so this is a NOOP.")
195197
fs.StringVar(&flags.CertDir, "cert-dir", "", "use certificates at the specified path to access the registry")
196198
fs.BoolVar(&flags.Compress, "compress", false, "this is a legacy option, which has no effect on the image")
@@ -265,6 +267,7 @@ func GetBudFlagsCompletions() commonComp.FlagCompletions {
265267
flagCompletion["annotation"] = commonComp.AutocompleteNone
266268
flagCompletion["authfile"] = commonComp.AutocompleteDefault
267269
flagCompletion["build-arg"] = commonComp.AutocompleteNone
270+
flagCompletion["build-context"] = commonComp.AutocompleteNone
268271
flagCompletion["cache-from"] = commonComp.AutocompleteNone
269272
flagCompletion["cert-dir"] = commonComp.AutocompleteDefault
270273
flagCompletion["creds"] = commonComp.AutocompleteNone

0 commit comments

Comments
 (0)