Transactions are tricky for deleting sources

tubesync/sync/management/commands/delete-source.py

@@ -2,18 +2,18 @@
 import uuid
 from django.utils.translation import gettext_lazy as _
 from django.core.management.base import BaseCommand, CommandError
-from django.db.models import signals
+from django.db.transaction import atomic
 from common.logger import log
 from sync.models import Source, Media, MediaServer
 from sync.tasks import schedule_media_servers_update
 
 
 class Command(BaseCommand):
 
-    help = ('Deletes a source by UUID')
+    help = _('Deletes a source by UUID')
 
     def add_arguments(self, parser):
-        parser.add_argument('--source', action='store', required=True, help='Source UUID')
+        parser.add_argument('--source', action='store', required=True, help=_('Source UUID'))
 
     def handle(self, *args, **options):
         source_uuid_str = options.get('source', '')
@@ -29,13 +29,15 @@ def handle(self, *args, **options):
             raise CommandError(f'Source does not exist with '
                                f'UUID: {source_uuid}')
         # Reconfigure the source to not update the disk or media servers
-        source.deactivate()
+        with atomic(durable=True):
+            source.deactivate()
         # Delete the source, triggering pre-delete signals for each media item
         log.info(f'Found source with UUID "{source.uuid}" with name '
                  f'"{source.name}" and deleting it, this may take some time!')
         log.info(f'Source directory: {source.directory_path}')
-        source.delete()
-        # Update any media servers
-        schedule_media_servers_update()
+        with atomic(durable=True):
+            source.delete()
+            # Update any media servers
+            schedule_media_servers_update()
         # All done
         log.info('Done')

tubesync/sync/signals.py

@@ -1,8 +1,9 @@
+from functools import partial
 from pathlib import Path
-from shutil import rmtree
 from tempfile import TemporaryDirectory
 from django.conf import settings
 from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
+from django.db.transaction import on_commit
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
 from background_task.signals import task_failed
@@ -20,6 +21,20 @@
 from .choices import Val, YouTube_SourceType
 
 
+def is_relative_to(self, *other):
+        """Return True if the path is relative to another path or False.
+        """
+        try:
+            self.relative_to(*other)
+            return True
+        except ValueError:
+            return False
+
+# patch Path for Python 3.8
+if not hasattr(Path, 'is_relative_to'):
+    Path.is_relative_to = is_relative_to
+
+
 @receiver(pre_save, sender=Source)
 def source_pre_save(sender, instance, **kwargs):
     # Triggered before a source is saved, if the schedule has been updated recreate
@@ -134,27 +149,30 @@ def source_post_save(sender, instance, created, **kwargs):
 def source_pre_delete(sender, instance, **kwargs):
     # Triggered before a source is deleted, delete all media objects to trigger
     # the Media models post_delete signal
+    source = instance
     log.info(f'Deactivating source: {instance.name}')
     instance.deactivate()
     log.info(f'Deleting tasks for source: {instance.name}')
     delete_task_by_source('sync.tasks.index_source_task', instance.pk)
     delete_task_by_source('sync.tasks.check_source_directory_exists', instance.pk)
     delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk)
     delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk)
-    # Schedule deletion of media
-    delete_task_by_source('sync.tasks.delete_all_media_for_source', instance.pk)
-    verbose_name = _('Deleting all media for source "{}"')
-    delete_all_media_for_source(
-        str(instance.pk),
-        str(instance.name),
-        verbose_name=verbose_name.format(instance.name),
-    )
-    # Try to do it all immediately
-    # If this is killed, the scheduled task should do the work instead.
-    delete_all_media_for_source.now(
-        str(instance.pk),
-        str(instance.name),
-    )
+
+    # Fetch the media source
+    sqs = Source.objects.filter(filter_text=str(source.pk))
+    if sqs.count():
+        media_source = sqs[0]
+        # Schedule deletion of media
+        delete_task_by_source('sync.tasks.delete_all_media_for_source', media_source.pk)
+        verbose_name = _('Deleting all media for source "{}"')
+        on_commit(partial(
+            delete_all_media_for_source,
+            str(media_source.pk),
+            str(media_source.name),
+            str(media_source.directory_path),
+            priority=1,
+            verbose_name=verbose_name.format(media_source.name),
+        ))
 
 
 @receiver(post_delete, sender=Source)
@@ -164,14 +182,8 @@ def source_post_delete(sender, instance, **kwargs):
     log.info(f'Deleting tasks for removed source: {source.name}')
     delete_task_by_source('sync.tasks.index_source_task', instance.pk)
     delete_task_by_source('sync.tasks.check_source_directory_exists', instance.pk)
-    delete_task_by_source('sync.tasks.delete_all_media_for_source', instance.pk)
     delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk)
     delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk)
-    # Remove the directory, if the user requested that
-    directory_path = Path(source.directory_path)
-    if (directory_path / '.to_be_removed').is_file():
-        log.info(f'Deleting directory for: {source.name}: {directory_path}')
-        rmtree(directory_path, True)
 
 
 @receiver(task_failed, sender=Task)

tubesync/sync/tasks.py

@@ -10,8 +10,9 @@
 import uuid
 from io import BytesIO
 from hashlib import sha1
+from pathlib import Path
 from datetime import datetime, timedelta
-from shutil import copyfile
+from shutil import copyfile, rmtree
 from PIL import Image
 from django.conf import settings
 from django.core.files.base import ContentFile
@@ -792,8 +793,9 @@ def wait_for_media_premiere(media_id):
         if task:
             update_task_status(task, f'available in {hours(media.published - now)} hours')
 
-@background(schedule=dict(priority=1, run_at=300), queue=Val(TaskQueue.FS), remove_existing_tasks=False)
-def delete_all_media_for_source(source_id, source_name):
+
+@background(schedule=dict(priority=1, run_at=90), queue=Val(TaskQueue.FS), remove_existing_tasks=False)
+def delete_all_media_for_source(source_id, source_name, source_directory):
     source = None
     try:
         source = Source.objects.get(pk=source_id)
@@ -807,8 +809,21 @@ def delete_all_media_for_source(source_id, source_name):
     ).filter(
         source=source or source_id,
     )
-    for media in mqs:
-        log.info(f'Deleting media for source: {source_name} item: {media.name}')
-        with atomic():
-            media.delete()
+    with atomic(durable=True):
+        for media in mqs:
+            log.info(f'Deleting media for source: {source_name} item: {media.name}')
+            with atomic():
+                media.delete()
+    # Remove the directory, if the user requested that
+    directory_path = Path(source_directory)
+    remove = (
+        (source and source.delete_removed_media) or
+        (directory_path / '.to_be_removed').is_file()
+    )
+    if source:
+        with atomic(durable=True):
+            source.delete()
+    if remove:
+        log.info(f'Deleting directory for: {source_name}: {directory_path}')
+        rmtree(directory_path, True)
 

tubesync/sync/views.py

@@ -26,12 +26,12 @@
 from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm,
                     SkipMediaForm, EnableMediaForm, ResetTasksForm,
                     ConfirmDeleteMediaServerForm)
-from .utils import validate_url, delete_file, multi_key_sort
+from .utils import validate_url, delete_file, multi_key_sort, mkdir_p
 from .tasks import (map_task_to_instance, get_error_message,
                     get_source_completed_tasks, get_media_download_task,
                     delete_task_by_media, index_source_task,
                     check_source_directory_exists, migrate_queues)
-from .choices import (Val, MediaServerType, SourceResolution,
+from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule,
                         YouTube_SourceType, youtube_long_source_types,
                         youtube_help, youtube_validation_urls)
 from . import signals
@@ -411,11 +411,39 @@ class DeleteSourceView(DeleteView, FormMixin):
     context_object_name = 'source'
 
     def post(self, request, *args, **kwargs):
+        source = self.get_object()
+        media_source = dict(
+            uuid=None,
+            index_schedule=IndexSchedule.NEVER,
+            download_media=False,
+            index_videos=False,
+            index_streams=False,
+            filter_text=str(source.pk),
+        )
+        copy_fields = set(map(lambda f: f.name, source._meta.fields)) - set(media_source.keys())
+        for k, v in source.__dict__.items():
+            if k in copy_fields:
+                media_source[k] = v
+        media_source = Source(**media_source)
         delete_media_val = request.POST.get('delete_media', False)
         delete_media = True if delete_media_val is not False else False
+        # overload this boolean for our own use
+        media_source.delete_removed_media = delete_media
+        # adjust the directory and key on the source to be deleted
+        source.directory = source.directory + '/deleted'
+        source.key = source.key + '/deleted'
+        source.name = f'[Deleting] {source.name}'
+        source.save(update_fields={'directory', 'key', 'name'})
+        source.refresh_from_db()
+        # save the new media source now that it is not a duplicate
+        media_source.uuid = None
+        media_source.save()
+        media_source.refresh_from_db()
+        # switch the media to the new source instance
+        Media.objects.filter(source=source).update(source=media_source)
         if delete_media:
-            source = self.get_object()
-            directory_path = pathlib.Path(source.directory_path)
+            directory_path = pathlib.Path(media_source.directory_path)
+            mkdir_p(directory_path)
             (directory_path / '.to_be_removed').touch(exist_ok=True)
         return super().post(request, *args, **kwargs)