Skip to content

Commit 5ec5e33

Browse files
wjhunwjhun
committed
re-add support for TFS version 4 encodings
This commit returns support for reading and optionally writing TFS version 4 images. 'ops image' commands that read an existing image will read either version 4 or 5 images, and commands that create images may either accept a "-4" or "--tfsv4" commandline option or a "TFSv4" config bool to create a version 4 image. This support is intended to help ease the transition towards use of TFS version 5.
1 parent 1605ee4 commit 5ec5e33

File tree

8 files changed

+116
-47
lines changed

8 files changed

+116
-47
lines changed

‎cmd/flags_build_image.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type BuildImageCommandFlags struct {
1919
DisableArgsCopy bool
2020
CmdEnvs []string
2121
ImageName string
22+
TFSv4 bool
2223
Mounts []string
2324
TargetRoot string
2425
IPAddress string
@@ -55,6 +56,10 @@ func (flags *BuildImageCommandFlags) MergeToConfig(c *types.Config) (err error)
5556
c.RunConfig.ImageName = flags.ImageName
5657
}
5758

59+
if flags.TFSv4 {
60+
c.TFSv4 = true
61+
}
62+
5863
setNanosBaseImage(c)
5964

6065
if c.RunConfig.ImageName == "" && c.Program != "" {
@@ -146,6 +151,11 @@ func NewBuildImageCommandFlags(cmdFlags *pflag.FlagSet) (flags *BuildImageComman
146151
exitWithError(err.Error())
147152
}
148153

154+
flags.TFSv4, err = cmdFlags.GetBool("tfsv4")
155+
if err != nil {
156+
exitWithError(err.Error())
157+
}
158+
149159
flags.TargetRoot, err = cmdFlags.GetString("target-root")
150160
if err != nil {
151161
exitWithError(err.Error())
@@ -209,6 +219,7 @@ func PersistBuildImageCommandFlags(cmdFlags *pflag.FlagSet) {
209219
cmdFlags.StringArrayP("envs", "e", nil, "env arguments")
210220
cmdFlags.StringP("target-root", "r", "", "target root")
211221
cmdFlags.StringP("imagename", "i", "", "image name")
222+
cmdFlags.BoolP("tfsv4", "4", false, "use TFSv4")
212223
cmdFlags.StringArray("mounts", nil, "mount <volume_id:mount_path>")
213224
cmdFlags.StringArrayP("args", "a", nil, "command line arguments")
214225
cmdFlags.BoolP("disable-args-copy", "", false, "disable copying of files passed as arguments")

‎cmd/flags_pkg_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ func TestPkgFlagsMergeToConfig(t *testing.T) {
108108
RunConfig: types.RunConfig{
109109
Memory: "2G",
110110
},
111+
TFSv4: true,
111112
}
112113

113114
err = pkgFlags.MergeToConfig(c)
@@ -137,6 +138,7 @@ func TestPkgFlagsMergeToConfig(t *testing.T) {
137138
Memory: "2G",
138139
ImageName: lepton.GetOpsHome() + "/images/ops",
139140
},
141+
TFSv4: true,
140142
}
141143

142144
assert.Equal(t, expectedConfig, c)

‎fs/mkfs.go‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -130,14 +130,15 @@ var uefiFileBootaa64 = []byte{
130130

131131
// MkfsCommand wraps mkfs calls
132132
type MkfsCommand struct {
133-
bootPath string
134-
uefiPath string
135-
label string
136-
manifest *Manifest
137-
partitions bool
138-
size int64
139-
outPath string
140-
rootTfs *tfs
133+
bootPath string
134+
uefiPath string
135+
label string
136+
manifest *Manifest
137+
partitions bool
138+
size int64
139+
outPath string
140+
rootTfs *tfs
141+
oldEncoding bool
141142
}
142143

143144
// NewMkfsCommand returns an instance of MkfsCommand
@@ -210,6 +211,11 @@ func (m *MkfsCommand) SetLabel(label string) {
210211
m.label = label
211212
}
212213

214+
// SetOldEncoding forces use of TFS version 4
215+
func (m *MkfsCommand) SetOldEncoding() {
216+
m.oldEncoding = true
217+
}
218+
213219
// Execute runs mkfs command
214220
func (m *MkfsCommand) Execute() error {
215221
if m.outPath == "" {
@@ -269,7 +275,7 @@ func (m *MkfsCommand) Execute() error {
269275
if manifest != nil {
270276
manifest.finalize()
271277
if manifest.boot != nil {
272-
_, err = tfsWrite(outFile, outOffset, bootFSSize, "", manifest.boot)
278+
_, err = tfsWrite(outFile, outOffset, bootFSSize, "", manifest.boot, m.oldEncoding)
273279
if err != nil {
274280
return fmt.Errorf("cannot write boot filesystem: %v", err)
275281
}
@@ -279,7 +285,7 @@ func (m *MkfsCommand) Execute() error {
279285
} else {
280286
root = mkFS()
281287
}
282-
m.rootTfs, err = tfsWrite(outFile, outOffset, 0, m.label, root)
288+
m.rootTfs, err = tfsWrite(outFile, outOffset, 0, m.label, root, m.oldEncoding)
283289
if err != nil {
284290
return fmt.Errorf("cannot write root filesystem: %v", err)
285291
}

‎fs/tfs.go‎

Lines changed: 75 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616

1717
const tfsMagic = "NVMTFS"
1818
const tfsVersion = 5
19+
const oldTfsVersion = 4
1920

2021
const logExtensionSize = 1024 * sectorSize
2122

@@ -62,28 +63,33 @@ type tfs struct {
6263
root *map[string]interface{}
6364
}
6465

65-
func (t *tfs) logInit() error {
66+
func (t *tfs) logInit(oldEncoding bool) error {
6667
if (t.size != 0) && (t.allocated+sectorSize > t.size) {
6768
return fmt.Errorf("available space (%d bytes) too small, required %d", t.size-t.allocated, sectorSize)
6869
}
69-
t.currentExt = t.newLogExt(true)
70+
t.currentExt = t.newLogExt(true, oldEncoding)
7071
t.allocated += sectorSize
7172
return t.logExtend()
7273
}
7374

74-
func (t *tfs) newLogExt(initial bool) *tlogExt {
75+
func (t *tfs) newLogExt(initial bool, oldEncoding bool) *tlogExt {
7576
var extSize uint
7677
if initial {
7778
extSize = sectorSize
7879
} else {
7980
extSize = logExtensionSize
8081
}
8182
logExt := &tlogExt{
82-
offset: t.allocated,
83-
buffer: make([]byte, 0, extSize),
83+
offset: t.allocated,
84+
oldEncoding: oldEncoding,
85+
buffer: make([]byte, 0, extSize),
8486
}
8587
logExt.buffer = append(logExt.buffer, tfsMagic...)
86-
logExt.buffer = appendVarint(logExt.buffer, tfsVersion)
88+
version := uint(tfsVersion)
89+
if oldEncoding {
90+
version = oldTfsVersion
91+
}
92+
logExt.buffer = appendVarint(logExt.buffer, version)
8793
logExt.buffer = appendVarint(logExt.buffer, extSize/sectorSize)
8894
if initial {
8995
logExt.buffer = append(logExt.buffer, t.uuid[:]...)
@@ -97,7 +103,7 @@ func (t *tfs) logExtend() error {
97103
if (t.size != 0) && (t.allocated+logExtensionSize > t.size) {
98104
return fmt.Errorf("available space (%d bytes) too small, required %d", t.size-t.allocated, logExtensionSize)
99105
}
100-
logExt := t.newLogExt(false)
106+
logExt := t.newLogExt(false, t.currentExt.oldEncoding)
101107
t.currentExt.linkTo(logExt.offset)
102108
t.allocated += logExtensionSize
103109
err := t.currentExt.flush(t.imgFile, t.imgOffset)
@@ -121,7 +127,12 @@ func (t *tfs) readLogExt(offset, size uint64) (uint64, error) {
121127
if err != nil {
122128
return 0, err
123129
}
124-
if version != tfsVersion {
130+
var oldEncoding bool
131+
if version == 4 {
132+
oldEncoding = true
133+
} else if version == tfsVersion {
134+
oldEncoding = false
135+
} else {
125136
return 0, fmt.Errorf("TFS version mismatch: expected %d, found %d", tfsVersion, version)
126137
}
127138
_size, err := getVarint(buffer, &offset)
@@ -170,7 +181,7 @@ func (t *tfs) readLogExt(offset, size uint64) (uint64, error) {
170181
}
171182
if length == tupleTotalLen {
172183
var decoded uint64
173-
_, err := t.decodeValue(buffer[offset:offset+uint64(length)], &decoded)
184+
_, err := t.decodeValue(buffer[offset:offset+uint64(length)], &decoded, oldEncoding)
174185
if err != nil {
175186
return 0, err
176187
}
@@ -194,7 +205,7 @@ func (t *tfs) readLogExt(offset, size uint64) (uint64, error) {
194205
t.decoder.tupleRemain -= length
195206
if t.decoder.tupleRemain == 0 {
196207
var decoded uint64
197-
_, err := t.decodeValue(t.staging, &decoded)
208+
_, err := t.decodeValue(t.staging, &decoded, oldEncoding)
198209
if err != nil {
199210
return 0, err
200211
}
@@ -252,20 +263,38 @@ func (t *tfs) writeDirEntries(dir map[string]interface{}) error {
252263
func (t *tfs) encodeValue(value interface{}) error {
253264
str, isStr := value.(string)
254265
if isStr {
255-
t.encodeString(str, typeString)
266+
strType := byte(typeString)
267+
if t.currentExt.oldEncoding {
268+
strType = typeBuffer
269+
}
270+
t.encodeString(str, strType)
256271
return nil
257272
}
258273
strSlice, isStrSlice := value.([]string)
259274
if isStrSlice {
260-
vector := make([]interface{}, len(strSlice))
275+
if !t.currentExt.oldEncoding {
276+
vector := make([]interface{}, len(strSlice))
277+
for i, str := range strSlice {
278+
vector[i] = str
279+
}
280+
return t.encodeVector(vector)
281+
}
282+
tuple := make(map[string]interface{})
261283
for i, str := range strSlice {
262-
vector[i] = str
284+
tuple[strconv.Itoa(i)] = str
263285
}
264-
return t.encodeVector(vector)
286+
return t.encodeTuple(tuple)
265287
}
266288
slice, isSlice := value.([]interface{})
267289
if isSlice {
268-
return t.encodeVector(slice)
290+
if !t.currentExt.oldEncoding {
291+
return t.encodeVector(slice)
292+
}
293+
tuple := make(map[string]interface{})
294+
for i, val := range slice {
295+
tuple[strconv.Itoa(i)] = val
296+
}
297+
return t.encodeTuple(tuple)
269298
}
270299
tuple, isTuple := value.(map[string]interface{})
271300
if isTuple {
@@ -323,19 +352,19 @@ func (t *tfs) encodeVector(vector []interface{}) error {
323352
return nil
324353
}
325354

326-
func (t *tfs) decodeValue(buffer []byte, offset *uint64) (interface{}, error) {
355+
func (t *tfs) decodeValue(buffer []byte, offset *uint64, oldEncoding bool) (interface{}, error) {
327356
var entry, dataType byte
328-
length, err := getHeader(buffer, offset, &entry, &dataType)
357+
length, err := getHeader(buffer, offset, &entry, &dataType, oldEncoding)
329358
if err != nil {
330359
return nil, err
331360
}
332361
switch dataType {
333362
case typeTuple:
334-
return t.decodeTuple(buffer, offset, entry, length)
363+
return t.decodeTuple(buffer, offset, entry, length, oldEncoding)
335364
case typeBuffer:
336365
return t.decodeBuf(buffer, offset, entry, length)
337366
case typeVector:
338-
return t.decodeVector(buffer, offset, entry, length)
367+
return t.decodeVector(buffer, offset, entry, length, oldEncoding)
339368
case typeInteger:
340369
return t.decodeInteger(buffer, offset, entry, length)
341370
case typeString:
@@ -345,7 +374,7 @@ func (t *tfs) decodeValue(buffer []byte, offset *uint64) (interface{}, error) {
345374
}
346375
}
347376

348-
func (t *tfs) decodeVector(buffer []byte, offset *uint64, entry byte, length uint) (*[]interface{}, error) {
377+
func (t *tfs) decodeVector(buffer []byte, offset *uint64, entry byte, length uint, oldEncoding bool) (*[]interface{}, error) {
349378
var vector *[]interface{}
350379
if entry == entryImmediate {
351380
newVector := make([]interface{}, length)
@@ -362,7 +391,7 @@ func (t *tfs) decodeVector(buffer []byte, offset *uint64, entry byte, length uin
362391
}
363392
}
364393
for i := 0; i < int(length); i++ {
365-
value, err := t.decodeValue(buffer, offset)
394+
value, err := t.decodeValue(buffer, offset, oldEncoding)
366395
if err != nil {
367396
return vector, err
368397
}
@@ -371,7 +400,7 @@ func (t *tfs) decodeVector(buffer []byte, offset *uint64, entry byte, length uin
371400
return vector, nil
372401
}
373402

374-
func (t *tfs) decodeTuple(buffer []byte, offset *uint64, entry byte, length uint) (*map[string]interface{}, error) {
403+
func (t *tfs) decodeTuple(buffer []byte, offset *uint64, entry byte, length uint, oldEncoding bool) (*map[string]interface{}, error) {
375404
var tuple *map[string]interface{}
376405
if entry == entryImmediate {
377406
newTuple := make(map[string]interface{})
@@ -389,7 +418,7 @@ func (t *tfs) decodeTuple(buffer []byte, offset *uint64, entry byte, length uint
389418
}
390419
for i := 0; i < int(length); i++ {
391420
var nameEntry, nameType byte
392-
n, err := getHeader(buffer, offset, &nameEntry, &nameType)
421+
n, err := getHeader(buffer, offset, &nameEntry, &nameType, oldEncoding)
393422
if err != nil {
394423
return tuple, err
395424
}
@@ -400,7 +429,7 @@ func (t *tfs) decodeTuple(buffer []byte, offset *uint64, entry byte, length uint
400429
if err != nil {
401430
return tuple, err
402431
}
403-
value, err := t.decodeValue(buffer, offset)
432+
value, err := t.decodeValue(buffer, offset, oldEncoding)
404433
if err != nil {
405434
return tuple, err
406435
}
@@ -543,12 +572,17 @@ func (t *tfs) pushHeader(entry byte, dataType byte, length int) {
543572
len64 := uint64(length)
544573
bitCount := uint(64 - bits.LeadingZeros64(len64))
545574
var words uint
546-
if bitCount > 3 {
547-
words = ((bitCount - 3) + (7 - 1)) / 7
575+
immBits := uint(3)
576+
if t.currentExt.oldEncoding {
577+
immBits = 5
548578
}
549-
var first = (entry << 7) | (dataType << 4) | byte(len64>>(words*7))
579+
if bitCount > immBits {
580+
words = ((bitCount - immBits) + (7 - 1)) / 7
581+
}
582+
var first = (entry << 7) | (dataType << (immBits + 1)) |
583+
byte(len64>>(words*7))
550584
if words != 0 {
551-
first |= 1 << 3
585+
first |= 1 << immBits
552586
}
553587
t.staging = append(t.staging, first)
554588
i := words
@@ -921,8 +955,9 @@ func (r *tfsFileReader) selectExtent(index int) {
921955
}
922956

923957
type tlogExt struct {
924-
offset uint64
925-
buffer []byte
958+
offset uint64
959+
oldEncoding bool
960+
buffer []byte
926961
}
927962

928963
func (e *tlogExt) linkTo(extOffset uint64) {
@@ -958,16 +993,20 @@ func appendVarint(buffer []byte, x uint) []byte {
958993
return buffer
959994
}
960995

961-
func getHeader(buffer []byte, offset *uint64, entry *byte, dataType *byte) (uint, error) {
996+
func getHeader(buffer []byte, offset *uint64, entry *byte, dataType *byte, oldEncoding bool) (uint, error) {
962997
if int(*offset) >= len(buffer) {
963998
return 0, fmt.Errorf("getHeader(): buffer length %d exhausted", len(buffer))
964999
}
9651000
b := buffer[*offset]
9661001
*offset++
9671002
*entry = b >> 7
968-
*dataType = (b >> 4) & 0x7
969-
length := uint(b & 0x7)
970-
if (b & (1 << 3)) != 0 {
1003+
immBits := uint(3)
1004+
if oldEncoding {
1005+
immBits = 5
1006+
}
1007+
*dataType = (b >> (immBits + 1)) & ((1 << (6 - immBits)) - 1)
1008+
length := uint(b & ((1 << immBits) - 1))
1009+
if (b & (1 << immBits)) != 0 {
9711010
for {
9721011
if int(*offset) >= len(buffer) {
9731012
return 0, fmt.Errorf("getHeader(): buffer length %d exhausted", len(buffer))
@@ -1042,15 +1081,15 @@ func newTfs(imgFile *os.File, imgOffset uint64, fsSize uint64) *tfs {
10421081
}
10431082

10441083
// tfsWrite writes filesystem metadata and contents to image file
1045-
func tfsWrite(imgFile *os.File, imgOffset uint64, fsSize uint64, label string, root map[string]interface{}) (*tfs, error) {
1084+
func tfsWrite(imgFile *os.File, imgOffset uint64, fsSize uint64, label string, root map[string]interface{}, oldEncoding bool) (*tfs, error) {
10461085
tfs := newTfs(imgFile, imgOffset, fsSize)
10471086
tfs.label = label
10481087
rand.Seed(time.Now().UnixNano())
10491088
_, err := rand.Read(tfs.uuid[:])
10501089
if err != nil {
10511090
return nil, fmt.Errorf("error generating random uuid: %v", err)
10521091
}
1053-
err = tfs.logInit()
1092+
err = tfs.logInit(oldEncoding)
10541093
if err != nil {
10551094
return nil, fmt.Errorf("cannot create filesystem log: %v", err)
10561095
}

‎fs/tfs_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ func TestLogExt(t *testing.T) {
1010
size: 512,
1111
}
1212
t.Run("initial log ext", func(t *testing.T) {
13-
ext := tfs.newLogExt(true)
13+
ext := tfs.newLogExt(true, false)
1414
if string(ext.buffer[:6]) != "NVMTFS" {
1515
t.Errorf("invalid TFS magic: got %d want 'NVMTFS'", ext.buffer[:6])
1616
}

0 commit comments

Comments
 (0)