F #6857: Add flattening support for Ceph incremental backups (#3699)

Signed-off-by: josepselga <[email protected]>
Co-authored-by: Guillemo Ramos <[email protected]>

Author: josepselga

install.sh

@@ -1314,6 +1314,7 @@ TM_LIB_FILES="src/tm_mad/lib/kvm.rb \
               src/tm_mad/lib/shell.rb \
               src/tm_mad/lib/tm_action.rb \
               src/tm_mad/lib/backup_qcow2.rb \
+              src/tm_mad/lib/backup_rbd.rb \
               src/tm_mad/lib/datastore.rb \
               src/tm_mad/lib/tm_cache.rb \
               src/tm_mad/lib/backup.rb"

src/datastore_mad/remotes/restic/backup

@@ -159,12 +159,13 @@ script = <<~EOS
     set -e -o pipefail; shopt -qs failglob
 
     #{rds.resticenv_sh}
+    BKNAME=$(ls #{vm_dir}/disk.* | head -n 1)
 
     RC=`#{ds.cmd_confinement(rcmd, vm_dir, ['SSH_AUTH_SOCK', 'RESTIC_PASSWORD', 'GOMAXPROCS'])}`
 
     INFO=`echo "$RC" | jq -r '.snapshot_id + " " + (.total_bytes_processed|tostring) | select ( . != " null" )'`
 
-    echo $INFO
+    echo "$INFO $BKNAME"
 EOS
 
 rc = TransferManager::Action.ssh 'backup',
@@ -181,7 +182,7 @@ end
 
 parts = rc.stdout.lines.last.split
 
-if parts.length != 2
+if parts.length != 3
     STDERR.puts "Wrong format for backup action: #{rc.stdout}"
     exit(-1)
 end
@@ -191,14 +192,9 @@ end
 id       = parts[0]
 short_id = id[0..7] # first 8 chars only
 
-vm            = REXML::Document.new(xml).root.elements['VM']
-backup_format =
-    if vm.elements['TEMPLATE/TM_MAD_SYSTEM'].text == 'ceph' &&
-       vm.elements['BACKUPS/BACKUP_CONFIG/MODE']&.text == 'INCREMENT'
-        'rbd'
-    else
-        'raw'
-    end
+disk_filepath = parts[2]
+extension     = File.extname(disk_filepath).delete_prefix('.')
+backup_format = ['rbd2', 'rbdiff'].include?(extension) ? 'rbd' : 'raw'
 
 STDOUT.puts "#{short_id} #{parts[1].to_i / (1024 * 1024)} #{backup_format}"
 exit(0)

src/datastore_mad/remotes/rsync/backup

@@ -159,10 +159,9 @@ script = <<~EOS
     set -ex -o pipefail
 
     BKSIZE=`du -sm #{vm_dir}`
-
+    BKNAME=$(ls #{vm_dir}/disk.* | head -n 1)
     #{ds.cmd_confinement(cmd, vm_dir, ['SSH_AUTH_SOCK'])} > /dev/null
-
-    echo $BKSIZE
+    echo "$BKSIZE $BKNAME"
 EOS
 
 rc = TransferManager::Action.ssh 'backup',
@@ -177,14 +176,11 @@ if rc.code != 0 || rc.stdout.empty?
     exit(-1)
 end
 
-vm            = REXML::Document.new(xml).root.elements['VM']
-backup_format =
-    if vm.elements['TEMPLATE/TM_MAD_SYSTEM'].text == 'ceph' &&
-       vm.elements['BACKUPS/BACKUP_CONFIG/MODE']&.text == 'INCREMENT'
-        'rbd'
-    else
-        'raw'
-    end
+stdout_parts = rc.stdout.split(' ')
+
+disk_filepath = stdout_parts[2]
+extension     = File.extname(disk_filepath).delete_prefix('.')
+backup_format = ['rbd2', 'rbdiff'].include?(extension) ? 'rbd' : 'raw'
 
 STDOUT.puts "#{backup_id} #{rc.stdout.lines.last.split[0]} #{backup_format}"
 exit(0)

src/tm_mad/lib/backup.rb

@@ -17,6 +17,8 @@
 #--------------------------------------------------------------------------- #
 
 require 'CommandManager'
+require 'rexml/document'
+
 require_relative 'kvm'
 require_relative 'shell'
 
@@ -42,14 +44,32 @@ def backup_disks_sh(options = {})
 
             snap_cmd = ''
             expo_cmd = ''
+
             snap_clup = ''
             expo_clup = ''
 
+            xml_vm = REXML::Document.new(@xml).root
+
+            bk_img_id_elem = xml_vm.elements['BACKUPS/BACKUP_IDS/ID']
+            bk_img_id      = bk_img_id_elem&.text&.strip
+
+            if bk_img_id.nil? || bk_img_id.empty?
+                disk_format = 'qcow2'
+            else
+                client = OpenNebula::Client.new
+                bk_img = OpenNebula::Image.new_with_id(bk_img_id, client)
+
+                bk_img.info
+
+                disk_format = bk_img['FORMAT']&.strip
+            end
+
             @disks.compact.each do |d|
                 did = d.id
                 next unless disks.include? did.to_s
 
-                cmds = d.backup_cmds(backup_dir, ds, live)
+                # Pass the format for this disk (or nil if not set)
+                cmds = d.backup_cmds(backup_dir, ds, live, disk_format)
                 return nil unless cmds
 
                 snap_cmd  << cmds[:snapshot].to_s