build: allow using cache explicitly with --squash-all using --layers

cmd/podman/images/build.go

@@ -197,9 +197,8 @@ func buildFlags(cmd *cobra.Command) {
 // build executes the build command.
 func build(cmd *cobra.Command, args []string) error {
 	if (cmd.Flags().Changed("squash") && cmd.Flags().Changed("layers")) ||
-		(cmd.Flags().Changed("squash-all") && cmd.Flags().Changed("layers")) ||
 		(cmd.Flags().Changed("squash-all") && cmd.Flags().Changed("squash")) {
-		return errors.New("cannot specify --squash, --squash-all and --layers options together")
+		return errors.New("cannot specify --squash with --layers and --squash-all with --squash")
 	}
 
 	if cmd.Flag("output").Changed && registry.IsRemote() {
@@ -418,7 +417,13 @@ func buildFlagsWrapperToOptions(c *cobra.Command, contextDir string, flags *buil
 	// Squash-all invoked, squash both new and old layers into one.
 	if c.Flags().Changed("squash-all") {
 		flags.Squash = true
-		flags.Layers = false
+		if !c.Flags().Changed("layers") {
+			// Buildah  supports using layers and --squash together
+			// after https://github.com/containers/buildah/pull/3674
+			// so podman must honor if user wants to still use layers
+			//  with --squash-all.
+			flags.Layers = false
+		}
 	}
 
 	var stdin io.Reader
@@ -442,22 +447,6 @@ func buildFlagsWrapperToOptions(c *cobra.Command, contextDir string, flags *buil
 		return nil, err
 	}
 
-	// `buildah bud --layers=false` acts like `docker build --squash` does.
-	// That is all of the new layers created during the build process are
-	// condensed into one, any layers present prior to this build are retained
-	// without condensing.  `buildah bud --squash` squashes both new and old
-	// layers down into one.  Translate Podman commands into Buildah.
-	// Squash invoked, retain old layers, squash new layers into one.
-	if c.Flags().Changed("squash") && flags.Squash {
-		flags.Squash = false
-		flags.Layers = false
-	}
-	// Squash-all invoked, squash both new and old layers into one.
-	if c.Flags().Changed("squash-all") {
-		flags.Squash = true
-		flags.Layers = false
-	}
-
 	compression := buildahDefine.Gzip
 	if flags.DisableCompression {
 		compression = buildahDefine.Uncompressed
@@ -513,9 +502,26 @@ func buildFlagsWrapperToOptions(c *cobra.Command, contextDir string, flags *buil
 		return nil, errors.Wrapf(err, "unable to obtain decrypt config")
 	}
 
+	additionalBuildContext := make(map[string]*buildahDefine.AdditionalBuildContext)
+	if c.Flag("build-context").Changed {
+		for _, contextString := range flags.BuildContext {
+			av := strings.SplitN(contextString, "=", 2)
+			if len(av) > 1 {
+				parseAdditionalBuildContext, err := parse.GetAdditionalBuildContext(av[1])
+				if err != nil {
+					return nil, errors.Wrapf(err, "while parsing additional build context")
+				}
+				additionalBuildContext[av[0]] = &parseAdditionalBuildContext
+			} else {
+				return nil, fmt.Errorf("while parsing additional build context: %q, accepts value in the form of key=value", av)
+			}
+		}
+	}
+
 	opts := buildahDefine.BuildOptions{
 		AddCapabilities:         flags.CapAdd,
 		AdditionalTags:          tags,
+		AdditionalBuildContexts: additionalBuildContext,
 		AllPlatforms:            flags.AllPlatforms,
 		Annotations:             flags.Annotation,
 		Args:                    args,
@@ -525,6 +531,7 @@ func buildFlagsWrapperToOptions(c *cobra.Command, contextDir string, flags *buil
 		Compression:             compression,
 		ConfigureNetwork:        networkPolicy,
 		ContextDirectory:        contextDir,
+		CPPFlags:                flags.CPPFlags,
 		DefaultMountsFilePath:   containerConfig.Containers.DefaultMountsFile,
 		Devices:                 flags.Devices,
 		DropCapabilities:        flags.CapDrop,

docs/source/markdown/podman-build.1.md

@@ -91,6 +91,33 @@ instructions read from the Containerfiles in the same way that environment
 variables are, but which will not be added to environment variable list in the
 resulting image's configuration.
 
+#### **--build-context**=*name=value*
+
+Specify an additional build context using its short name and its location.
+Additional build contexts can be referenced in the same manner as we access
+different stages in COPY instruction.
+
+Valid values could be:
+
+* Local directory – e.g. --build-context project2=../path/to/project2/src
+* HTTP URL to a tarball – e.g. --build-context src=https://example.org/releases/src.tar
+* Container image – specified with a container-image:// prefix, e.g. --build-context alpine=container-image://alpine:3.15, (also accepts docker://, docker-image://)
+
+On the Containerfile side, you can reference the build context on all
+commands that accept the “from” parameter. Here’s how that might look:
+
+```dockerfile
+FROM [name]
+COPY --from=[name] ...
+RUN --mount=from=[name] …
+```
+
+The value of [name] is matched with the following priority order:
+
+* Named build context defined with --build-context [name]=..
+* Stage defined with AS [name] inside Containerfile
+* Image [name], either local or in a remote registry
+
 #### **--cache-from**
 
 Images to utilize as potential cache sources. Podman does not currently support
@@ -140,6 +167,10 @@ This option is added to be aligned with other containers CLIs.
 Podman doesn't communicate with a daemon or a remote server.
 Thus, compressing the data before sending it is irrelevant to Podman. (This option is not available with the remote Podman client, including Mac and Windows (excluding WSL2) machines)
 
+#### **--cpp-flag**=*flags*
+
+Set additional flags to pass to the C Preprocessor cpp(1). Containerfiles ending with a ".in" suffix will be preprocessed via cpp(1). This option can be used to pass additional flags to cpp.Note: You can also set default CPPFLAGS by setting the BUILDAH_CPPFLAGS environment variable (e.g., export BUILDAH_CPPFLAGS="-DDEBUG").
+
 #### **--cpu-period**=*limit*
 
 Set the CPU period for the Completely Fair Scheduler (CFS), which is a

go.mod

@@ -11,13 +11,13 @@ require (
 	github.com/container-orchestrated-devices/container-device-interface v0.4.0
 	github.com/containernetworking/cni v1.1.0
 	github.com/containernetworking/plugins v1.1.1
-	github.com/containers/buildah v1.26.1
+	github.com/containers/buildah v1.26.1-0.20220524184833-5500333c2e06
 	github.com/containers/common v0.48.1-0.20220523155016-2fd37da97824
 	github.com/containers/conmon v2.0.20+incompatible
-	github.com/containers/image/v5 v5.21.2-0.20220519193817-1e26896b8059
+	github.com/containers/image/v5 v5.21.2-0.20220520105616-e594853d6471
 	github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f
 	github.com/containers/psgo v1.7.2
-	github.com/containers/storage v1.41.1-0.20220511210719-cacc3325a9c8
+	github.com/containers/storage v1.41.1-0.20220517121726-5019cd55275c
 	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3
 	github.com/cyphar/filepath-securejoin v0.2.3

go.sum

@@ -268,7 +268,6 @@ github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0
 github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
 github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
 github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE=
-github.com/containerd/containerd v1.6.3/go.mod h1:gCVGrYRYFm2E8GmuUIbj/NGD7DLZQLzSJQazjVKDOig=
 github.com/containerd/containerd v1.6.4 h1:SEDZBp10mhCp+hkO3Njz/YhGrI7ah3edNcUlRdUPOgg=
 github.com/containerd/containerd v1.6.4/go.mod h1:oWOqbuJUZmOVafhA0lj2NAXbiO1u7F0K5l1bUgdyo94=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -290,7 +289,6 @@ github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZH
 github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
 github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
 github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
-github.com/containerd/go-cni v1.1.4/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
 github.com/containerd/go-cni v1.1.5/go.mod h1:Rf2ZrMycr1El589IyuRzn7RkfdRZVKaFGaxSDHVAjj0=
 github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
@@ -337,17 +335,16 @@ github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRD
 github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE=
 github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE=
 github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8=
-github.com/containers/buildah v1.26.1 h1:D65Vuo+orsI14WWtJhSX6KrpgBBa7+hveVWevzG8p8E=
-github.com/containers/buildah v1.26.1/go.mod h1:CsWSG8OpJd8v3mlLREJzVAOBgC93DjRNALUVHoi8QsY=
-github.com/containers/common v0.48.0/go.mod h1:zPLZCfLXfnd1jI0QRsD4By54fP4k1+ifQs+tulIe3o0=
+github.com/containers/buildah v1.26.1-0.20220524184833-5500333c2e06 h1:Tx1IfKch/SnsCk1YrdyR4B2AcS1TKLYxbSMXzmQXafU=
+github.com/containers/buildah v1.26.1-0.20220524184833-5500333c2e06/go.mod h1:oB0PwsW+rhePNsBimCnEz4YMLx8QxZBjHi/DPnXhUCg=
+github.com/containers/common v0.48.1-0.20220519181648-280c6f69fa82/go.mod h1:Ru/JjL1CTHzlxghVMhchzcFUwHLvlIeR5/SUMw8VUOI=
 github.com/containers/common v0.48.1-0.20220523155016-2fd37da97824 h1:5gMIUUpIK9DvHrrlj1Tik8GfCh5DEuVqm0JnYHWYUDw=
 github.com/containers/common v0.48.1-0.20220523155016-2fd37da97824/go.mod h1:Ru/JjL1CTHzlxghVMhchzcFUwHLvlIeR5/SUMw8VUOI=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
-github.com/containers/image/v5 v5.21.1/go.mod h1:zl35egpcDQa79IEXIuoUe1bW+D1pdxRxYjNlyb3YiXw=
 github.com/containers/image/v5 v5.21.2-0.20220511203756-fe4fd4ed8be4/go.mod h1:OsX9sFexyGF0FCNAjfcVFv3IwMqDyLyV/WQY/roLPcE=
-github.com/containers/image/v5 v5.21.2-0.20220519193817-1e26896b8059 h1:/FzsjrQ2nJtMom9IXEGieORlwUk/NyDuuz5SWcNo324=
-github.com/containers/image/v5 v5.21.2-0.20220519193817-1e26896b8059/go.mod h1:KntCBNQn3qOuZmQuJ38ORyTozmWXiuo05Vef2S0Sm5M=
+github.com/containers/image/v5 v5.21.2-0.20220520105616-e594853d6471 h1:2mm1jEFATvpdFfp8lUB/yc237OqwruMvfIPiVn1Wpgg=
+github.com/containers/image/v5 v5.21.2-0.20220520105616-e594853d6471/go.mod h1:KntCBNQn3qOuZmQuJ38ORyTozmWXiuo05Vef2S0Sm5M=
 github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a h1:spAGlqziZjCJL25C6F1zsQY05tfCKE9F5YwtEWWe6hU=
 github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
 github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
@@ -361,11 +358,10 @@ github.com/containers/psgo v1.7.2 h1:WbCvsY9w+nCv3j4der0mbD3PSRUv/W8l+G0YrZrdSDc
 github.com/containers/psgo v1.7.2/go.mod h1:SLpqxsPOHtTqRygjutCPXmeU2PoEFzV3gzJplN4BMx0=
 github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4=
 github.com/containers/storage v1.38.0/go.mod h1:lBzt28gAk5ADZuRtwdndRJyqX22vnRaXmlF+7ktfMYc=
-github.com/containers/storage v1.40.0/go.mod h1:zUyPC3CFIGR1OhY1CKkffxgw9+LuH76PGvVcFj38dgs=
 github.com/containers/storage v1.40.2/go.mod h1:zUyPC3CFIGR1OhY1CKkffxgw9+LuH76PGvVcFj38dgs=
 github.com/containers/storage v1.41.0/go.mod h1:Pb0l5Sm/89kolX3o2KolKQ5cCHk5vPNpJrhNaLcdS5s=
-github.com/containers/storage v1.41.1-0.20220511210719-cacc3325a9c8 h1:4XdTbn3iVIr1+kN5srZND2G3/Q3hJiZSZZtKdL6r9jg=
-github.com/containers/storage v1.41.1-0.20220511210719-cacc3325a9c8/go.mod h1:Pb0l5Sm/89kolX3o2KolKQ5cCHk5vPNpJrhNaLcdS5s=
+github.com/containers/storage v1.41.1-0.20220517121726-5019cd55275c h1:DQVf7UhxndNUtZ2+BIS/GtEdzszxMxrdqe43DRKRV2w=
+github.com/containers/storage v1.41.1-0.20220517121726-5019cd55275c/go.mod h1:HjV2DQuTFnjKYXDS3foE1EHODXu+dKHi7gT+uxT+kNk=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -431,7 +427,6 @@ github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v20.10.3-0.20220208084023-a5c757555091+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v20.10.15+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v20.10.16+incompatible h1:2Db6ZR/+FUR3hqPMwnogOPHFn405crbpxvWzKovETOQ=
 github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=

pkg/api/handlers/compat/images_build.go

@@ -80,6 +80,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) {
 		CgroupParent           string   `schema:"cgroupparent"` // nolint
 		Compression            uint64   `schema:"compression"`
 		ConfigureNetwork       string   `schema:"networkmode"`
+		CPPFlags               string   `schema:"cppflags"`
 		CpuPeriod              uint64   `schema:"cpuperiod"`  // nolint
 		CpuQuota               int64    `schema:"cpuquota"`   // nolint
 		CpuSetCpus             string   `schema:"cpusetcpus"` // nolint
@@ -399,6 +400,15 @@ func BuildImage(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	// convert cppflags formats
+	var cppflags = []string{}
+	if _, found := r.URL.Query()["cppflags"]; found {
+		if err := json.Unmarshal([]byte(query.CPPFlags), &cppflags); err != nil {
+			utils.BadRequest(w, "cppflags", query.CPPFlags, err)
+			return
+		}
+	}
+
 	// convert nsoptions formats
 	nsoptions := buildah.NamespaceOptions{}
 	if _, found := r.URL.Query()["nsoptions"]; found {
@@ -555,6 +565,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) {
 		AddCapabilities: addCaps,
 		AdditionalTags:  additionalTags,
 		Annotations:     annotations,
+		CPPFlags:        cppflags,
 		Args:            buildArgs,
 		AllPlatforms:    query.AllPlatforms,
 		CommonBuildOpts: &buildah.CommonBuildOptions{

pkg/bindings/images/build.go

@@ -65,6 +65,14 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
 		params.Set("annotations", l)
 	}
 
+	if cppflags := options.CPPFlags; len(cppflags) > 0 {
+		l, err := jsoniter.MarshalToString(cppflags)
+		if err != nil {
+			return nil, err
+		}
+		params.Set("cppflags", l)
+	}
+
 	if options.AllPlatforms {
 		params.Add("allplatforms", "1")
 	}

test/buildah-bud/apply-podman-deltas

@@ -193,6 +193,22 @@ skip_if_remote "volumes don't work with podman-remote" \
                "buildah bud --volume" \
                "buildah-bud-policy"
 
+# Most of this should work in podman remote after API implementation other than where context is host.
+skip_if_remote "--build-context option not implemented in podman-remote" \
+               "build-with-additional-build-context and COPY, test pinning image" \
+               "build-with-additional-build-context and COPY, stagename and additional-context conflict" \
+               "build-with-additional-build-context and COPY, additionalContext and numeric value of stage" \
+               "build-with-additional-build-context and COPY, additionalContext and numeric value of stage" \
+               "build-with-additional-build-context and COPY, additional context from host" \
+               "build-with-additional-build-context and COPY, additional context from external URL" \
+               "build-with-additional-build-context and RUN --mount=from=, additional-context is URL and mounted from subdir" \
+               "build-with-additional-build-context and RUN --mount=from=, additional-context not image and also test conflict with stagename" \
+               "build-with-additional-build-context and RUN --mount=from=, additional-context and also test conflict with stagename" \
+               "bud-multiple-platform for --all-platform with additional-build-context" \
+               "build-with-additional-build-context and FROM, stagename and additional-context conflict" \
+               "bud with Containerfile.in, via envariable" \
+               "build-with-additional-build-context and FROM, pin busybox to alpine"
+
 # Requires a local file outside context dir
 skip_if_remote "local keyfile not sent to podman-remote" \
                "bud with encrypted FROM image"

test/e2e/build_test.go

@@ -178,6 +178,32 @@ var _ = Describe("Podman build", func() {
 		Expect(session).Should(Exit(0))
 	})
 
+	It("podman build verify explicit cache use with squash-all and --layers", func() {
+		session := podmanTest.Podman([]string{"build", "--pull-never", "-f", "build/squash/Dockerfile.squash-c", "--squash-all", "--layers", "-t", "test-squash-d:latest", "build/squash"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"inspect", "--format", "{{.RootFS.Layers}}", "test-squash-d"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		// Check for one layers
+		Expect(strings.Fields(session.OutputToString())).To(HaveLen(1))
+
+		// Second build must use last squashed build from cache
+		session = podmanTest.Podman([]string{"build", "--pull-never", "-f", "build/squash/Dockerfile.squash-c", "--squash-all", "--layers", "-t", "test", "build/squash"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		// Test if entire build is used from cache
+		Expect(session.OutputToString()).To(ContainSubstring("Using cache"))
+
+		session = podmanTest.Podman([]string{"inspect", "--format", "{{.RootFS.Layers}}", "test-squash-d"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		// Check for one layers
+		Expect(strings.Fields(session.OutputToString())).To(HaveLen(1))
+
+	})
+
 	It("podman build Containerfile locations", func() {
 		// Given
 		// Switch to temp dir and restore it afterwards