Merge branch 'main' into fix-legacy-fixtures-minimal

Author: danielhollas

codecov.yml

@@ -15,3 +15,6 @@ coverage:
     patch:
       default:
         threshold: 0.1%
+
+ignore:
+- src/aiida/tools/_dumping/**/*

docs/source/howto/data.rst

@@ -223,6 +223,7 @@ Below is a list with all available subcommands.
       create        Create an empty group with a given label.
       delete        Delete groups and (optionally) the nodes they contain.
       description   Change the description of a group.
+      dump          Dump data of an AiiDA group to disk.
       list          Show a list of existing groups.
       move-nodes    Move the specified NODES from one group to another.
       path          Inspect groups of nodes, with delimited label paths.
@@ -397,6 +398,7 @@ Below is a list with all available subcommands.
     Commands:
       configure-rabbitmq  Configure RabbitMQ for a profile.
       delete              Delete one or more profiles.
+      dump                Dump all data in an AiiDA profile's storage to disk.
       list                Display a list of all available profiles.
       set-default         Set a profile as the default profile.
       setdefault          (Deprecated) Set a profile as the default profile.
@@ -428,8 +430,8 @@ Below is a list with all available subcommands.
       --first-name NONEMPTYSTRING     First name of the user.  [required]
       --last-name NONEMPTYSTRING      Last name of the user.  [required]
       --institution NONEMPTYSTRING    Institution of the user.  [required]
-      --db-engine [postgresql_psycopg]
-                                      Engine to use to connect to the database.  [required]
+      --db-engine [postgresql_psycopg|postgresql_psycopg2]
+                                      Engine to use to connect to the database. (deprecated)
       --db-backend [core.psql_dos]    Database backend to use.  [required]
       --db-host HOSTNAME              Database server host. Leave empty for "peer"
                                       authentication.  [required]
@@ -535,8 +537,8 @@ Below is a list with all available subcommands.
       --first-name NONEMPTYSTRING     First name of the user.  [required]
       --last-name NONEMPTYSTRING      Last name of the user.  [required]
       --institution NONEMPTYSTRING    Institution of the user.  [required]
-      --db-engine [postgresql_psycopg]
-                                      Engine to use to connect to the database.  [required]
+      --db-engine [postgresql_psycopg|postgresql_psycopg2]
+                                      Engine to use to connect to the database. (deprecated)
       --db-backend [core.psql_dos]    Database backend to use.  [required]
       --db-host HOSTNAME              Database server host. Leave empty for "peer"
                                       authentication.  [required]

docs/source/reference/command_line.rst

@@ -279,6 +279,11 @@ Documentation = 'https://aiida.readthedocs.io'
 Home = 'http://www.aiida.net/'
 Source = 'https://github.com/aiidateam/aiida-core'
 
+[tool.coverage.run]
+omit = [
+  "src/aiida/tools/_dumping/**/*"
+]
+
 [tool.flit.module]
 name = 'aiida'
 

pyproject.toml

@@ -137,7 +137,7 @@ def group_move_nodes(source_group, target_group, force, nodes, all_entries):
 
     if not force:
         click.confirm(
-            f'Are you sure you want to move {len(nodes)} nodes from {source_group} ' f'to {target_group}?', abort=True
+            f'Are you sure you want to move {len(nodes)} nodes from {source_group} to {target_group}?', abort=True
         )
 
     source_group.remove_nodes(nodes)
@@ -325,6 +325,11 @@ def group_relabel(group, label):
         echo.echo_critical(str(exception))
     else:
         echo.echo_success(f"Label changed to '{label}'")
+        msg = (
+            'Note that if you are dumping your profile data to disk, to reflect the relabeling of the group, '
+            'run your `verdi profile dump` command again.'
+        )
+        echo.echo_report(msg)
 
 
 @verdi_group.command('description')
@@ -632,3 +637,106 @@ def group_path_ls(path, type_string, recursive, as_table, no_virtual, with_descr
             if no_virtual and child.is_virtual:
                 continue
             echo.echo(child.path, bold=not child.is_virtual)
+
+
+@verdi_group.command('dump')
+@arguments.GROUP()
+@options.PATH()
+@options.DRY_RUN()
+@options.OVERWRITE()
+@options.PAST_DAYS()
+@options.START_DATE()
+@options.END_DATE()
+@options.FILTER_BY_LAST_DUMP_TIME()
+@options.ONLY_TOP_LEVEL_CALCS()
+@options.ONLY_TOP_LEVEL_WORKFLOWS()
+@options.DELETE_MISSING()
+@options.SYMLINK_CALCS()
+@options.INCLUDE_INPUTS()
+@options.INCLUDE_OUTPUTS()
+@options.INCLUDE_ATTRIBUTES()
+@options.INCLUDE_EXTRAS()
+@options.FLAT()
+@options.DUMP_UNSEALED()
+@click.pass_context
+@with_dbenv()
+def group_dump(
+    ctx,
+    group,
+    path,
+    dry_run,
+    overwrite,
+    past_days,
+    start_date,
+    end_date,
+    filter_by_last_dump_time,
+    delete_missing,
+    only_top_level_calcs,
+    only_top_level_workflows,
+    symlink_calcs,
+    include_inputs,
+    include_outputs,
+    include_attributes,
+    include_extras,
+    flat,
+    dump_unsealed,
+):
+    """Dump data of an AiiDA group to disk."""
+
+    import traceback
+    from pathlib import Path
+
+    from aiida.cmdline.utils import echo
+    from aiida.tools._dumping.utils import DumpPaths
+
+    warning_msg = (
+        'This is a new feature which is still in its testing phase. '
+        'If you encounter unexpected behavior or bugs, please report them via Discourse or GitHub.'
+    )
+    echo.echo_warning(warning_msg)
+
+    try:
+        if path is None:
+            group_path = DumpPaths.get_default_dump_path(group)
+            dump_base_output_path = Path.cwd() / group_path
+            echo.echo_report(f'No output path specified. Using default: `{dump_base_output_path}`')
+        else:
+            dump_base_output_path = Path(path).resolve()
+            echo.echo_report(f'Using specified output path: `{dump_base_output_path}`')
+
+        # --- Logical Checks ---
+        if dry_run and overwrite:
+            msg = (
+                '`--dry-run` and `--overwrite` selected (or set in config). Overwrite operation will NOT be performed.'
+            )
+            echo.echo_warning(msg)
+
+        # Run the dumping
+        group.dump(
+            output_path=dump_base_output_path,
+            dry_run=dry_run,
+            overwrite=overwrite,
+            past_days=past_days,
+            start_date=start_date,
+            end_date=end_date,
+            filter_by_last_dump_time=filter_by_last_dump_time,
+            only_top_level_calcs=only_top_level_calcs,
+            only_top_level_workflows=only_top_level_workflows,
+            symlink_calcs=symlink_calcs,
+            include_inputs=include_inputs,
+            include_outputs=include_outputs,
+            include_attributes=include_attributes,
+            include_extras=include_extras,
+            flat=flat,
+            dump_unsealed=dump_unsealed,
+        )
+
+        if not dry_run:
+            msg = f'Raw files for group `{group.label}` dumped into folder `{dump_base_output_path.name}`.'
+            echo.echo_success(msg)
+        else:
+            echo.echo_success('Dry run completed.')
+
+    except Exception as e:
+        msg = f'Unexpected error during dump of group {group.label}:\n ({e!s}).\n'
+        echo.echo_critical(msg + traceback.format_exc())

src/aiida/cmdline/commands/cmd_group.py

@@ -14,6 +14,7 @@
 from aiida.cmdline.params import arguments, options, types
 from aiida.cmdline.params.options.overridable import OverridableOption
 from aiida.cmdline.utils import decorators, echo
+from aiida.cmdline.utils.decorators import with_dbenv
 from aiida.common.log import LOG_LEVELS, capture_logging
 
 REPAIR_INSTRUCTIONS = """\
@@ -583,98 +584,94 @@ def process_repair(manager, broker, dry_run):
 @verdi_process.command('dump')
 @arguments.PROCESS()
 @options.PATH()
+@options.DRY_RUN()
 @options.OVERWRITE()
-@click.option(
-    '--include-inputs/--exclude-inputs',
-    default=True,
-    show_default=True,
-    help='Include the linked input nodes of the `CalculationNode`(s).',
-)
-@click.option(
-    '--include-outputs/--exclude-outputs',
-    default=False,
-    show_default=True,
-    help='Include the linked output nodes of the `CalculationNode`(s).',
-)
-@click.option(
-    '--include-attributes/--exclude-attributes',
-    default=True,
-    show_default=True,
-    help='Include attributes in the `.aiida_node_metadata.yaml` written for every `ProcessNode`.',
-)
-@click.option(
-    '--include-extras/--exclude-extras',
-    default=True,
-    show_default=True,
-    help='Include extras in the `.aiida_node_metadata.yaml` written for every `ProcessNode`.',
-)
-@click.option(
-    '-f',
-    '--flat',
-    is_flag=True,
-    default=False,
-    show_default=True,
-    help='Dump files in a flat directory for every step of the workflow.',
-)
-@click.option(
-    '--dump-unsealed',
-    is_flag=True,
-    default=False,
-    show_default=True,
-    help='Also allow the dumping of unsealed process nodes.',
-)
-@options.INCREMENTAL()
+@options.INCLUDE_INPUTS()
+@options.INCLUDE_OUTPUTS()
+@options.INCLUDE_ATTRIBUTES()
+@options.INCLUDE_EXTRAS()
+@options.FLAT()
+@options.DUMP_UNSEALED()
+@click.pass_context
+@with_dbenv()
 def process_dump(
+    ctx,
     process,
     path,
+    dry_run,
     overwrite,
     include_inputs,
     include_outputs,
     include_attributes,
     include_extras,
     flat,
     dump_unsealed,
-    incremental,
 ) -> None:
     """Dump process input and output files to disk.
 
     Child calculations/workflows (also called `CalcJob`s/`CalcFunction`s and `WorkChain`s/`WorkFunction`s in AiiDA
     jargon) run by the parent workflow are contained in the directory tree as sub-folders and are sorted by their
-    creation time.  The directory tree thus mirrors the logical execution of the workflow, which can also be queried by
+    creation time. The directory tree thus mirrors the logical execution of the workflow, which can also be queried by
     running `verdi process status <pk>` on the command line.
 
     By default, input and output files of each calculation can be found in the corresponding "inputs" and
     "outputs" directories (the former also contains the hidden ".aiida" folder with machine-readable job execution
     settings). Additional input and output files (depending on the type of calculation) are placed in the "node_inputs"
     and "node_outputs", respectively.
 
-    Lastly, every folder also contains a hidden, human-readable `.aiida_node_metadata.yaml` file with the relevant AiiDA
+    Lastly, every folder also contains a hidden, human-readable `aiida_node_metadata.yaml` file with the relevant AiiDA
     node data for further inspection.
     """
+    import traceback
+    from pathlib import Path
 
+    from aiida.cmdline.utils import echo
+    from aiida.tools._dumping.utils import DumpPaths
     from aiida.tools.archive.exceptions import ExportValidationError
-    from aiida.tools.dumping.processes import ProcessDumper
-
-    process_dumper = ProcessDumper(
-        include_inputs=include_inputs,
-        include_outputs=include_outputs,
-        include_attributes=include_attributes,
-        include_extras=include_extras,
-        overwrite=overwrite,
-        flat=flat,
-        dump_unsealed=dump_unsealed,
-        incremental=incremental,
+
+    warning_msg = (
+        'This is a new feature which is still in its testing phase. '
+        'If you encounter unexpected behavior or bugs, please report them via Discourse or GitHub.'
     )
+    echo.echo_warning(warning_msg)
 
+    # Check for dry_run + overwrite
+    if overwrite and dry_run:
+        msg = 'Both `dry_run` and `overwrite` set to true. Operation will NOT be performed.'
+        echo.echo_warning(msg)
+        return
+
+    if path is None:
+        process_path = DumpPaths.get_default_dump_path(process)
+        dump_base_output_path = Path.cwd() / process_path
+        msg = f'No output path specified. Using default: `{dump_base_output_path}`'
+        echo.echo_report(msg)
+    else:
+        echo.echo_report(f'Using specified output path: `{path}`')
+        dump_base_output_path = Path(path).resolve()
+
+    if dry_run:
+        echo.echo_success('Dry run completed.')
+        return
+
+    # Execute dumping
     try:
-        dump_path = process_dumper.dump(process_node=process, output_path=path)
-    except FileExistsError:
-        echo.echo_critical(
-            'Dumping directory exists and overwrite is False. Set overwrite to True, or delete directory manually.'
+        process.dump(
+            output_path=dump_base_output_path,
+            dry_run=dry_run,
+            overwrite=overwrite,
+            include_inputs=include_inputs,
+            include_outputs=include_outputs,
+            include_attributes=include_attributes,
+            include_extras=include_extras,
+            flat=flat,
+            dump_unsealed=dump_unsealed,
         )
+
+        msg = f'Raw files for process `{process.pk}` dumped into folder `{dump_base_output_path.name}`.'
+        echo.echo_success(msg)
     except ExportValidationError as e:
-        echo.echo_critical(f'{e!s}')
+        echo.echo_critical(f'Data validation error during dump: {e!s}')
     except Exception as e:
-        echo.echo_critical(f'Unexpected error while dumping {process.__class__.__name__} <{process.pk}>:\n ({e!s}).')
-
-    echo.echo_success(f'Raw files for {process.__class__.__name__} <{process.pk}> dumped into folder `{dump_path}`.')
+        msg = f'Unexpected error during dump of process {process.pk}:\n ({e!s}).\n'
+        echo.echo_critical(msg + traceback.format_exc())