Skip to content

Commit 585e4dd

Browse files
committed
buildkit: supports additionalBuildContext in builds via --build-context
As builds got more complicated, the ability to only access files from one location became quite limiting. With `multi-stage` builds where you can `copy` files from other parts of the Containerfile by adding the `--from` flag and pointing it to the name of another Containerfile stage or a remote image. The new named build context feature is an extension of this pattern. You can now define additional build contexts when running the build command, give them a name, and then access them inside a Dockerfile the same way you previously did with build stages. Additional build contexts can be defined with a new `--build-context [name]=[value]` flag. The key component defines the name for your build context and the value can be: ```console Local directory – e.g. --build-context project2=../path/to/project2/src HTTP URL to a tarball – e.g. --build-context src=https://example.org/releases/src.tar Container image – Define with a docker-image:// prefix, e.g. --build-context alpine=docker-image://alpine:3.15, ( also supports docker://, container-image:// ) ``` On the Containerfile side, you can reference the build context on all commands that accept the “from” parameter. Here’s how that might look: ```Dockerfile FROM [name] COPY --from=[name] ... RUN --mount=from=[name] … ``` The value of [name] is matched with the following priority order: * Named build context defined with `--build-context [name]=..` * Stage defined with `AS [name]` inside Dockerfile * Remote image `[name]` in a container registry Added Features * Pinning images for `FROM` and `COPY` * Specifying multiple buildcontexts from different projects and using them with `--from` in `ADD` and `COPY` directive * Override a Remote Dependency with a Local One. * Using additional context from external `Tar` Signed-off-by: Aditya R <[email protected]>
1 parent f671e53 commit 585e4dd

File tree

12 files changed

+474
-24
lines changed

12 files changed

+474
-24
lines changed

cmd/buildah/build.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,20 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error {
157157
}
158158
}
159159

160+
additionalBuildContext := make(map[string]*define.AdditionalBuildContext)
161+
if c.Flag("build-context").Changed {
162+
for _, contextString := range iopts.BuildContext {
163+
av := strings.SplitN(contextString, "=", 2)
164+
if len(av) > 1 {
165+
parseAdditionalBuildContext, err := parse.GetAdditionalBuildContext(av[1])
166+
if err != nil {
167+
return errors.Wrapf(err, "while parsing additional build context")
168+
}
169+
additionalBuildContext[av[0]] = &parseAdditionalBuildContext
170+
}
171+
}
172+
}
173+
160174
containerfiles := getContainerfiles(iopts.File)
161175
format, err := getFormat(iopts.Format)
162176
if err != nil {
@@ -344,6 +358,7 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error {
344358
Annotations: iopts.Annotation,
345359
Architecture: systemContext.ArchitectureChoice,
346360
Args: args,
361+
AdditionalBuildContexts: additionalBuildContext,
347362
BlobDirectory: iopts.BlobCache,
348363
CNIConfigDir: iopts.CNIConfigDir,
349364
CNIPluginPath: iopts.CNIPlugInPath,

define/build.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,21 @@ import (
1111
"golang.org/x/sync/semaphore"
1212
)
1313

14+
// AdditionalBuildContext contains verbose details about a parsed build context from --build-context
15+
type AdditionalBuildContext struct {
16+
// Value is the URL of an external tar archive.
17+
IsURL bool
18+
// Value is the name of an image which may or may not have already been pulled.
19+
IsImage bool
20+
// Value holds a URL, an image name, or an absolute filesystem path.
21+
Value string
22+
// Absolute filesystem path to downloaded and exported build context
23+
// from external tar archive. This will be populated only if following
24+
// buildcontext is created from IsURL and was downloaded before in any
25+
// of the RUN step.
26+
DownloadedCache string
27+
}
28+
1429
// CommonBuildOptions are resources that can be defined by flags for both buildah from and build
1530
type CommonBuildOptions struct {
1631
// AddHost is the list of hostnames to add to the build container's /etc/hosts.
@@ -121,6 +136,8 @@ type BuildOptions struct {
121136
Compression archive.Compression
122137
// Arguments which can be interpolated into Dockerfiles
123138
Args map[string]string
139+
// Map of external additional build contexts
140+
AdditionalBuildContexts map[string]*AdditionalBuildContext
124141
// Name of the image to write to.
125142
Output string
126143
// BuildOutput specifies if any custom build output is selected for following build.

docs/buildah-build.1.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,32 @@ resulting image's configuration.
6868
Please refer to the [BUILD TIME VARIABLES](#build-time-variables) section for the
6969
list of variables that can be overridden within the Containerfile at run time.
7070

71+
**--build-context** *name=value*
72+
73+
Specify an additional build context using its short name and its location. Additional
74+
build contexts can be referenced in the same manner as we access different stages in `COPY`
75+
instruction.
76+
77+
Valid values could be:
78+
* Local directory – e.g. --build-context project2=../path/to/project2/src
79+
* HTTP URL to a tarball – e.g. --build-context src=https://example.org/releases/src.tar
80+
* Container image – specified with a container-image:// prefix, e.g. --build-context alpine=container-image://alpine:3.15, ( also accepts docker://, docker-image://)
81+
82+
On the Containerfile side, you can reference the build context on all commands that accept the “from” parameter.
83+
Here’s how that might look:
84+
85+
```Dockerfile
86+
FROM [name]
87+
COPY --from=[name] ...
88+
RUN --mount=from=[name] …
89+
```
90+
91+
The value of `[name]` is matched with the following priority order:
92+
93+
* Named build context defined with --build-context [name]=..
94+
* Stage defined with AS [name] inside Containerfile
95+
* Image [name], either local or in a remote registry
96+
7197
**--cache-from**
7298

7399
Images to utilise as potential cache sources. Buildah does not currently support --cache-from so this is a NOOP.

imagebuildah/build.go

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,12 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B
211211
}
212212

213213
if options.AllPlatforms {
214-
options.Platforms, err = platformsForBaseImages(ctx, logger, paths, files, options.From, options.Args, options.SystemContext)
214+
additionalBuildContexts := options.AdditionalBuildContexts
215+
if additionalBuildContexts == nil {
216+
// Create empty map for additionalbuildContexts
217+
additionalBuildContexts = make(map[string]*define.AdditionalBuildContext)
218+
}
219+
options.Platforms, err = platformsForBaseImages(ctx, logger, paths, files, options.From, options.Args, additionalBuildContexts, options.SystemContext)
215220
if err != nil {
216221
return "", nil, err
217222
}
@@ -502,8 +507,8 @@ func preprocessContainerfileContents(logger *logrus.Logger, containerfile string
502507
// platformsForBaseImages resolves the names of base images from the
503508
// dockerfiles, and if they are all valid references to manifest lists, returns
504509
// the list of platforms that are supported by all of the base images.
505-
func platformsForBaseImages(ctx context.Context, logger *logrus.Logger, dockerfilepaths []string, dockerfiles [][]byte, from string, args map[string]string, systemContext *types.SystemContext) ([]struct{ OS, Arch, Variant string }, error) {
506-
baseImages, err := baseImages(dockerfilepaths, dockerfiles, from, args)
510+
func platformsForBaseImages(ctx context.Context, logger *logrus.Logger, dockerfilepaths []string, dockerfiles [][]byte, from string, args map[string]string, additionalBuildContext map[string]*define.AdditionalBuildContext, systemContext *types.SystemContext) ([]struct{ OS, Arch, Variant string }, error) {
511+
baseImages, err := baseImages(logger, dockerfilepaths, dockerfiles, from, args, additionalBuildContext)
507512
if err != nil {
508513
return nil, errors.Wrapf(err, "determining list of base images")
509514
}
@@ -631,7 +636,7 @@ func platformsForBaseImages(ctx context.Context, logger *logrus.Logger, dockerfi
631636
// stage's base image with FROM, and returns the list of base images as
632637
// provided. Each entry in the dockerfilenames slice corresponds to a slice in
633638
// dockerfilecontents.
634-
func baseImages(dockerfilenames []string, dockerfilecontents [][]byte, from string, args map[string]string) ([]string, error) {
639+
func baseImages(logger *logrus.Logger, dockerfilenames []string, dockerfilecontents [][]byte, from string, args map[string]string, additionalBuildContext map[string]*define.AdditionalBuildContext) ([]string, error) {
635640
mainNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(dockerfilecontents[0]))
636641
if err != nil {
637642
return nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfilenames[0])
@@ -670,6 +675,13 @@ func baseImages(dockerfilenames []string, dockerfilecontents [][]byte, from stri
670675
child.Next.Value = from
671676
from = ""
672677
}
678+
if replaceBuildContext, ok := additionalBuildContext[child.Next.Value]; ok {
679+
if replaceBuildContext.IsImage {
680+
child.Next.Value = replaceBuildContext.Value
681+
} else {
682+
logger.Warnf("additionalContext for nickname %q was found but its not an image hence it will be not used with FROM %q", child.Next.Value, child.Next.Value)
683+
}
684+
}
673685
base := child.Next.Value
674686
if base != "scratch" && !nicknames[base] {
675687
// TODO: this didn't undergo variable and arg

imagebuildah/executor.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ type Executor struct {
126126
imageInfoLock sync.Mutex
127127
imageInfoCache map[string]imageTypeAndHistoryAndDiffIDs
128128
fromOverride string
129+
additionalBuildContexts map[string]*define.AdditionalBuildContext
129130
manifest string
130131
secrets map[string]define.Secret
131132
sshsources map[string]*sshagent.Source
@@ -275,6 +276,7 @@ func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, o
275276
rusageLogFile: rusageLogFile,
276277
imageInfoCache: make(map[string]imageTypeAndHistoryAndDiffIDs),
277278
fromOverride: options.From,
279+
additionalBuildContexts: options.AdditionalBuildContexts,
278280
manifest: options.Manifest,
279281
secrets: secrets,
280282
sshsources: sshsources,
@@ -609,6 +611,12 @@ func (b *Executor) Build(ctx context.Context, stages imagebuilder.Stages) (image
609611
}
610612
base := child.Next.Value
611613
if base != "scratch" {
614+
if replaceBuildContext, ok := b.additionalBuildContexts[child.Next.Value]; ok {
615+
if replaceBuildContext != nil && replaceBuildContext.IsImage {
616+
child.Next.Value = replaceBuildContext.Value
617+
base = child.Next.Value
618+
}
619+
}
612620
userArgs := argsMapToSlice(stage.Builder.Args)
613621
baseWithArg, err := imagebuilder.ProcessWord(base, userArgs)
614622
if err != nil {

imagebuildah/stage_executor.go

Lines changed: 139 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -362,25 +362,78 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err
362362
stripSetgid := false
363363
preserveOwnership := false
364364
contextDir := s.executor.contextDir
365+
365366
if len(copy.From) > 0 {
367+
// If additionalContext is selected do not process anything else
368+
didSelectAdditionalContext := false
369+
// If additionalContext is selected but it was image only process
370+
// image and ignore stages if any. [ Buildx parity ]
371+
didSelectImageAsAdditionalContext := false
366372
// If from has an argument within it, resolve it to its
367373
// value. Otherwise just return the value found.
368374
from, fromErr := imagebuilder.ProcessWord(copy.From, s.stage.Builder.Arguments())
369375
if fromErr != nil {
370376
return errors.Wrapf(fromErr, "unable to resolve argument %q", copy.From)
371377
}
372-
if isStage, err := s.executor.waitForStage(s.ctx, from, s.stages[:s.index]); isStage && err != nil {
373-
return err
378+
if additionalBuildContext, ok := s.executor.additionalBuildContexts[from]; ok {
379+
if additionalBuildContext != nil {
380+
didSelectAdditionalContext = true
381+
if !additionalBuildContext.IsImage {
382+
contextDir = additionalBuildContext.Value
383+
if additionalBuildContext.IsURL {
384+
// Check if following buildContext was already downloaded before in any other RUN step.
385+
// If yes lets check if we are hitting any cache and cache actually exists on host.
386+
cacheExistsOnHost := false
387+
if additionalBuildContext.DownloadedCache != "" {
388+
if stat, err := os.Stat(additionalBuildContext.DownloadedCache); err == nil && stat.IsDir() {
389+
cacheExistsOnHost = true
390+
}
391+
}
392+
if !cacheExistsOnHost {
393+
// additional context contains a tar file
394+
// so download and explode tar to buildah
395+
// temp and point context to that.
396+
path, subdir, err := define.TempDirForURL(internalUtil.GetTempDir(), internal.BuildahExternalArtifactsDir, additionalBuildContext.Value)
397+
if err != nil {
398+
return errors.Wrapf(err, "unable to download context from external source %q", additionalBuildContext.Value)
399+
}
400+
// point context dir to the extracted path
401+
contextDir = filepath.Join(path, subdir)
402+
// populate cache for next RUN step
403+
additionalBuildContext.DownloadedCache = contextDir
404+
} else {
405+
contextDir = additionalBuildContext.DownloadedCache
406+
}
407+
}
408+
} else {
409+
didSelectImageAsAdditionalContext = true
410+
copy.From = additionalBuildContext.Value
411+
}
412+
}
374413
}
375-
if other, ok := s.executor.stages[from]; ok && other.index < s.index {
376-
contextDir = other.mountPoint
377-
idMappingOptions = &other.builder.IDMappingOptions
378-
} else if builder, ok := s.executor.containerMap[copy.From]; ok {
379-
contextDir = builder.MountPoint
380-
idMappingOptions = &builder.IDMappingOptions
381-
} else {
382-
return errors.Errorf("the stage %q has not been built", copy.From)
414+
if !didSelectAdditionalContext {
415+
if isStage, err := s.executor.waitForStage(s.ctx, from, s.stages[:s.index]); isStage && err != nil {
416+
return err
417+
}
418+
if other, ok := s.executor.stages[from]; ok && other.index < s.index {
419+
contextDir = other.mountPoint
420+
idMappingOptions = &other.builder.IDMappingOptions
421+
} else if builder, ok := s.executor.containerMap[copy.From]; ok {
422+
contextDir = builder.MountPoint
423+
idMappingOptions = &builder.IDMappingOptions
424+
} else {
425+
return errors.Errorf("the stage %q has not been built", copy.From)
426+
}
427+
} else if didSelectImageAsAdditionalContext {
428+
// Image was selected as additionalContext so only process image.
429+
if builder, ok := s.executor.containerMap[copy.From]; ok {
430+
contextDir = builder.MountPoint
431+
idMappingOptions = &builder.IDMappingOptions
432+
} else {
433+
return errors.Errorf("the image %q is not mounted", copy.From)
434+
}
383435
}
436+
// Original behaviour of buildah still stays true for COPY irrespective of additional context.
384437
preserveOwnership = true
385438
copyExcludes = excludes
386439
} else {
@@ -446,6 +499,61 @@ func (s *StageExecutor) runStageMountPoints(mountList []string) (map[string]inte
446499
if fromErr != nil {
447500
return nil, errors.Wrapf(fromErr, "unable to resolve argument %q", kv[1])
448501
}
502+
// If additional buildContext contains this
503+
// give priority to that and break if additional
504+
// is not an external image.
505+
if additionalBuildContext, ok := s.executor.additionalBuildContexts[from]; ok {
506+
if additionalBuildContext != nil {
507+
if additionalBuildContext.IsImage {
508+
mountPoint, err := s.getImageRootfs(s.ctx, additionalBuildContext.Value)
509+
if err != nil {
510+
return nil, errors.Errorf("%s from=%s: no stage or image found with that name", flag, from)
511+
}
512+
// The `from` in stageMountPoints should point
513+
// to `mountPoint` replaced from additional
514+
// build-context. Reason: Parser will use this
515+
// `from` to refer from stageMountPoints map later.
516+
stageMountPoints[from] = internal.StageMountDetails{IsStage: false, MountPoint: mountPoint}
517+
break
518+
} else {
519+
// Most likely this points to path on filesystem
520+
// or external tar archive, Treat it as a stage
521+
// nothing is different for this. So process and
522+
// point mountPoint to path on host and it will
523+
// be automatically handled correctly by since
524+
// GetBindMount will honor IsStage:false while
525+
// processing stageMountPoints.
526+
mountPoint := additionalBuildContext.Value
527+
if additionalBuildContext.IsURL {
528+
// Check if following buildContext was already downloaded before in any other RUN step.
529+
// If yes lets check if we are hitting any cache and cache actually exists on host.
530+
cacheExistsOnHost := false
531+
if additionalBuildContext.DownloadedCache != "" {
532+
if stat, err := os.Stat(additionalBuildContext.DownloadedCache); err == nil && stat.IsDir() {
533+
cacheExistsOnHost = true
534+
}
535+
}
536+
if !cacheExistsOnHost {
537+
// additional context contains a tar file
538+
// so download and explode tar to buildah
539+
// temp and point context to that.
540+
path, subdir, err := define.TempDirForURL(internalUtil.GetTempDir(), internal.BuildahExternalArtifactsDir, additionalBuildContext.Value)
541+
if err != nil {
542+
return nil, errors.Wrapf(err, "unable to download context from external source %q", additionalBuildContext.Value)
543+
}
544+
// point context dir to the extracted path
545+
mountPoint = filepath.Join(path, subdir)
546+
// populate cache for next RUN step
547+
additionalBuildContext.DownloadedCache = mountPoint
548+
} else {
549+
mountPoint = additionalBuildContext.DownloadedCache
550+
}
551+
}
552+
stageMountPoints[from] = internal.StageMountDetails{IsStage: true, MountPoint: mountPoint}
553+
break
554+
}
555+
}
556+
}
449557
// If the source's name corresponds to the
450558
// result of an earlier stage, wait for that
451559
// stage to finish being built.
@@ -923,6 +1031,27 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
9231031
if fromErr != nil {
9241032
return "", nil, errors.Wrapf(fromErr, "unable to resolve argument %q", arr[1])
9251033
}
1034+
// If additional buildContext contains this
1035+
// give priority to that and break if additional
1036+
// is not an external image.
1037+
if additionalBuildContext, ok := s.executor.additionalBuildContexts[from]; ok {
1038+
if additionalBuildContext != nil {
1039+
if !additionalBuildContext.IsImage {
1040+
// We don't need to pull this
1041+
// since this additional context
1042+
// is not an image.
1043+
break
1044+
} else {
1045+
// replace with image set in build context
1046+
from = additionalBuildContext.Value
1047+
if _, err := s.getImageRootfs(ctx, from); err != nil {
1048+
return "", nil, errors.Errorf("%s --from=%s: no stage or image found with that name", command, from)
1049+
}
1050+
break
1051+
}
1052+
}
1053+
}
1054+
9261055
// If the source's name corresponds to the
9271056
// result of an earlier stage, wait for that
9281057
// stage to finish being built.

internal/parse/parse.go

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ func GetCacheMount(args []string, store storage.Store, imageMountLabel string, a
309309
// add subdirectory if specified
310310

311311
// cache parent directory
312-
cacheParent := filepath.Join(getTempDir(), BuildahCacheDir)
312+
cacheParent := filepath.Join(internalUtil.GetTempDir(), BuildahCacheDir)
313313
// create cache on host if not present
314314
err = os.MkdirAll(cacheParent, os.FileMode(0755))
315315
if err != nil {
@@ -597,12 +597,3 @@ func GetTmpfsMount(args []string) (specs.Mount, error) {
597597

598598
return newMount, nil
599599
}
600-
601-
/* This is internal function and could be changed at any time */
602-
/* for external usage please refer to buildah/pkg/parse.GetTempDir() */
603-
func getTempDir() string {
604-
if tmpdir, ok := os.LookupEnv("TMPDIR"); ok {
605-
return tmpdir
606-
}
607-
return "/var/tmp"
608-
}

internal/types.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
package internal
22

3+
const (
4+
// Temp directory which stores external artifacts which are download for a build.
5+
// Example: tar files from external sources.
6+
BuildahExternalArtifactsDir = "buildah-external-artifacts"
7+
)
8+
39
// Types is internal packages are suspected to change with releases avoid using these outside of buildah
410

511
// StageMountDetails holds the Stage/Image mountpoint returned by StageExecutor

internal/util/util.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ func LookupImage(ctx *types.SystemContext, store storage.Store, image string) (*
3232
return localImage, nil
3333
}
3434

35+
// GetTempDir returns base for a temporary directory on host.
36+
func GetTempDir() string {
37+
if tmpdir, ok := os.LookupEnv("TMPDIR"); ok {
38+
return tmpdir
39+
}
40+
return "/var/tmp"
41+
}
42+
3543
// ExportFromReader reads bytes from given reader and exports to external tar, directory or stdout.
3644
func ExportFromReader(input io.Reader, opts define.BuildOutputOption) error {
3745
var err error

0 commit comments

Comments
 (0)