Skip to content

Commit 5d78dc5

Browse files
helsaawyhelsaawy
authored
Add rootfs.exe tool to merge tar image layers (#2424)
* Add `rootfs.exe` tool to merge tar image layers Add tool to merge layer tarball files, which is used for creating WCOW uVM images with additional features added, without first needing to create a dedicated container image. This basically amounts to the append and flatten container image operations from the [crane](github.com/google/go-containerregistry/cmd/crane) tool, with some specific changes: - use `"path"` and not `"path/filepath"` for path manipulation, which avoids paths with `\` separators; - append a trailing `/` to directory paths (since different `tar` implementations may append them); and - overwrite the owner UID and GID to zero, to avoid any user issues. The tool avoids needing to unpack tar files within the same directory (as is done for LCOW) to combine the deltas with tar images. Based on 2024 Hackathon to replace current LCOW uVM rootfs creation, before the switch to Azure Linux. Signed-off-by: Hamza El-Saawy <[email protected]> * PR: remove leftover code, fix linter, extra comment Signed-off-by: Hamza El-Saawy <[email protected]> --------- Signed-off-by: Hamza El-Saawy <[email protected]>
1 parent be13ee5 commit 5d78dc5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+11594
-44
lines changed

‎go.mod‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ require (
3131
github.com/pkg/errors v0.9.1
3232
github.com/sirupsen/logrus v1.9.3
3333
github.com/urfave/cli v1.22.15
34+
github.com/urfave/cli/v2 v2.27.5
3435
github.com/vishvananda/netlink v1.3.0
3536
github.com/vishvananda/netns v0.0.4
3637
go.etcd.io/bbolt v1.3.10
@@ -54,7 +55,7 @@ require (
5455
github.com/containerd/log v0.1.0 // indirect
5556
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
5657
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
57-
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
58+
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
5859
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
5960
github.com/docker/cli v24.0.0+incompatible // indirect
6061
github.com/docker/distribution v2.8.3+incompatible // indirect
@@ -101,6 +102,7 @@ require (
101102
github.com/x448/float16 v0.8.4 // indirect
102103
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
103104
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
105+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
104106
github.com/yashtewari/glob-intersection v0.2.0 // indirect
105107
go.opentelemetry.io/otel v1.31.0 // indirect
106108
go.opentelemetry.io/otel/metric v1.31.0 // indirect

‎go.sum‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,9 @@ github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfed
6363
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
6464
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
6565
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
66-
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
6766
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
67+
github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
68+
github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
6869
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6970
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
7071
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -259,6 +260,8 @@ github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwD
259260
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
260261
github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM=
261262
github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0=
263+
github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
264+
github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
262265
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
263266
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
264267
github.com/veraison/go-cose v1.1.0 h1:AalPS4VGiKavpAzIlBjrn7bhqXiXi4jbMYY/2+UC+4o=
@@ -273,6 +276,8 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMc
273276
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
274277
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
275278
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
279+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
280+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
276281
github.com/yashtewari/glob-intersection v0.2.0 h1:8iuHdN88yYuCzCdjt0gDe+6bAhUwBeEWqThExu54RFg=
277282
github.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok=
278283
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

‎internal/tools/rootfs/main.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Tool to merge Windows and Linux rootfs.tar(.gz) and delta.tar (or other files) into
2+
// a unified rootfs (gzipped) TAR.
3+
4+
package main
5+
6+
import (
7+
"fmt"
8+
"os"
9+
10+
"github.com/sirupsen/logrus"
11+
"github.com/urfave/cli/v2"
12+
"go.opencensus.io/trace"
13+
14+
"github.com/Microsoft/hcsshim/internal/oc"
15+
)
16+
17+
// TODO: add tests for:
18+
// - general functionality (Windows + Linux)
19+
// - adding `./` prefix
20+
// - adding `/` suffix
21+
// - overriding UID and GUID
22+
// TODO: output CPIO archive?
23+
24+
func main() {
25+
logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true})
26+
27+
trace.ApplyConfig(trace.Config{DefaultSampler: oc.DefaultSampler})
28+
trace.RegisterExporter(&oc.LogrusExporter{})
29+
30+
if err := run(); err != nil {
31+
fmt.Fprintln(os.Stderr, err)
32+
os.Exit(1)
33+
}
34+
}
35+
36+
func run() error {
37+
args := os.Args
38+
39+
mergeCommand, err := newMergeCommand()
40+
if err != nil {
41+
return fmt.Errorf("could not create merge command: %w", err)
42+
}
43+
44+
app := &cli.App{
45+
Name: "rootfs",
46+
Usage: "manipulate rootfs tar(.gz) files",
47+
48+
Flags: []cli.Flag{
49+
&cli.StringFlag{
50+
Name: "log-level",
51+
Aliases: []string{"lvl"},
52+
Usage: "logging `level`",
53+
Value: logrus.StandardLogger().Level.String(),
54+
Action: func(_ *cli.Context, s string) error {
55+
lvl, err := logrus.ParseLevel(s)
56+
if err == nil {
57+
logrus.SetLevel(lvl)
58+
}
59+
return err
60+
},
61+
},
62+
},
63+
64+
Commands: []*cli.Command{
65+
mergeCommand,
66+
},
67+
DefaultCommand: mergeCommand.Name,
68+
ExitErrHandler: func(ctx *cli.Context, err error) {
69+
if err != nil {
70+
logrus.WithFields(logrus.Fields{
71+
logrus.ErrorKey: err,
72+
"command": fmt.Sprintf("%#+v", args),
73+
}).Error(ctx.App.Name + " failed")
74+
}
75+
},
76+
}
77+
78+
return app.Run(args)
79+
}

0 commit comments

Comments
 (0)