Merge branch 'main' into fix-legacy-fixtures

Author: danielhollas

.github/workflows/ci-code.yml

@@ -138,7 +138,11 @@ jobs:
   test-pytest-fixtures:
     # Who watches the watchmen?
     # Here we test the pytest fixtures in isolation from the rest of aiida-core test suite,
+<<<<<<< fix-legacy-fixtures
     # since since they can be used outside of aiida core context, e.g. in plugins.
+=======
+    # since they can be used outside of aiida core context, e.g. in plugins.
+>>>>>>> main
     # Unlike in other workflows in this file, we purposefully don't setup a test profile.
 
     runs-on: ubuntu-24.04
@@ -174,6 +178,7 @@ jobs:
         from-lock: 'true'
         extras: tests
 
+<<<<<<< fix-legacy-fixtures
     - name: Setup SSH
       run: .github/workflows/setup_ssh.sh
 
@@ -182,6 +187,10 @@ jobs:
 
     - name: Test legacy pytest fixtures
       run: pytest -sv --cov aiida --noconftest src/aiida/manage/tests/test_pytest_fixtures.py
+=======
+    - name: Test legacy pytest fixtures
+      run: pytest --cov aiida --noconftest src/aiida/manage/tests/test_pytest_fixtures.py
+>>>>>>> main
 
     - name: Upload coverage report
       if: github.repository == 'aiidateam/aiida-core'

.pre-commit-config.yaml

@@ -126,8 +126,6 @@ repos:
         src/aiida/manage/configuration/__init__.py|
         src/aiida/manage/configuration/config.py|
         src/aiida/manage/external/rmq/launcher.py|
-        src/aiida/manage/tests/main.py|
-        src/aiida/manage/tests/pytest_fixtures.py|
         src/aiida/orm/comments.py|
         src/aiida/orm/computers.py|
         src/aiida/orm/implementation/storage_backend.py|

codecov.yml

@@ -15,3 +15,6 @@ coverage:
     patch:
       default:
         threshold: 0.1%
+
+ignore:
+- src/aiida/tools/_dumping/**/*

docs/source/howto/data.rst

@@ -223,6 +223,7 @@ Below is a list with all available subcommands.
       create        Create an empty group with a given label.
       delete        Delete groups and (optionally) the nodes they contain.
       description   Change the description of a group.
+      dump          Dump data of an AiiDA group to disk.
       list          Show a list of existing groups.
       move-nodes    Move the specified NODES from one group to another.
       path          Inspect groups of nodes, with delimited label paths.
@@ -397,6 +398,7 @@ Below is a list with all available subcommands.
     Commands:
       configure-rabbitmq  Configure RabbitMQ for a profile.
       delete              Delete one or more profiles.
+      dump                Dump all data in an AiiDA profile's storage to disk.
       list                Display a list of all available profiles.
       set-default         Set a profile as the default profile.
       setdefault          (Deprecated) Set a profile as the default profile.
@@ -428,8 +430,8 @@ Below is a list with all available subcommands.
       --first-name NONEMPTYSTRING     First name of the user.  [required]
       --last-name NONEMPTYSTRING      Last name of the user.  [required]
       --institution NONEMPTYSTRING    Institution of the user.  [required]
-      --db-engine [postgresql_psycopg]
-                                      Engine to use to connect to the database.  [required]
+      --db-engine [postgresql_psycopg|postgresql_psycopg2]
+                                      Engine to use to connect to the database. (deprecated)
       --db-backend [core.psql_dos]    Database backend to use.  [required]
       --db-host HOSTNAME              Database server host. Leave empty for "peer"
                                       authentication.  [required]
@@ -535,8 +537,8 @@ Below is a list with all available subcommands.
       --first-name NONEMPTYSTRING     First name of the user.  [required]
       --last-name NONEMPTYSTRING      Last name of the user.  [required]
       --institution NONEMPTYSTRING    Institution of the user.  [required]
-      --db-engine [postgresql_psycopg]
-                                      Engine to use to connect to the database.  [required]
+      --db-engine [postgresql_psycopg|postgresql_psycopg2]
+                                      Engine to use to connect to the database. (deprecated)
       --db-backend [core.psql_dos]    Database backend to use.  [required]
       --db-host HOSTNAME              Database server host. Leave empty for "peer"
                                       authentication.  [required]

docs/source/reference/command_line.rst

@@ -632,17 +632,46 @@ Using the ``COPY`` mode, the target path defines another location (on the same f
    }
 
 .. note::
+    In addition to the ``COPY`` mode, the following modes, these storage efficient modes are also are available:
+    ``COMPRESS_TAR``, ``COMPRESS_TARBZ2``, ``COMPRESS_TARGZ``, ``COMPRESS_TARXZ``.
 
-   In the future, other methods for stashing may be implemented, such as placing all files in a (compressed) tarball or even stash files on tape.
+The stashed files and folders are represented by an output node that is attached to the calculation node through the label ``remote_stash``, as a ``RemoteStashFolderData`` node.
+Just like the ``remote_folder`` node, this represents a location or files on a remote machine and so is equivalent to a "symbolic link".
 
 .. important::
 
-   If the ``stash`` option namespace is defined for a calculation job, the daemon will perform the stashing operations before the files are retrieved.
+   If the ``stash`` option namespace is defined for a generic calculation job, the daemon will perform the stashing operations before the files are retrieved.
    This means that the stashing happens before the parsing of the output files (which occurs after the retrieving step), such that that the files will be stashed independent of the final exit status that the parser will assign to the calculation job.
    This may cause files to be stashed for calculations that will later be considered to have failed.
 
-The stashed files and folders are represented by an output node that is attached to the calculation node through the label ``remote_stash``, as a ``RemoteStashFolderData`` node.
-Just like the ``remote_folder`` node, this represents a location or files on a remote machine and so is equivalent to a "symbolic link".
+To avoid this scenario, you can instead, stash via a separate calculation job, for example:
+
+.. code-block:: python
+
+    from aiida.common.datastructures import StashMode
+    from aiida.orm import load_node, load_computer
+
+    StashCalculation = CalculationFactory('core.stash')
+
+    calcjob_node = load_node(<CALCJOB_PK>)
+    remote_folder = calcjob_node.outputs.remote_folder
+
+    inputs = {
+        'metadata': {
+            'computer': load_computer(label="localhost"),
+            'options': {
+                'stash': {
+                'source_list': ['aiida.out', 'output.txt'],
+                'target_base': '/scratch/',
+                'stash_mode': StashMode.COPY.value,
+                },
+            },
+        },
+        'source_node': remote_folder,
+    }
+
+    result = run(StashCalculation, **inputs)
+
 
 .. important::
 

docs/source/topics/calculations/usage.rst

@@ -279,6 +279,11 @@ Documentation = 'https://aiida.readthedocs.io'
 Home = 'http://www.aiida.net/'
 Source = 'https://github.com/aiidateam/aiida-core'
 
+[tool.coverage.run]
+omit = [
+  "src/aiida/tools/_dumping/**/*"
+]
+
 [tool.flit.module]
 name = 'aiida'
 

pyproject.toml

@@ -26,7 +26,7 @@ class StashCalculation(CalcJob):
 
         inputs = {
             'metadata': {
-                'computer': Computer.collection.get(label="localhost"),
+                'computer': load_computer(label="localhost"),
                 'options': {
                     'resources': {'num_machines': 1},
                     'stash': {

src/aiida/calculations/stash.py

@@ -137,7 +137,7 @@ def group_move_nodes(source_group, target_group, force, nodes, all_entries):
 
     if not force:
         click.confirm(
-            f'Are you sure you want to move {len(nodes)} nodes from {source_group} ' f'to {target_group}?', abort=True
+            f'Are you sure you want to move {len(nodes)} nodes from {source_group} to {target_group}?', abort=True
         )
 
     source_group.remove_nodes(nodes)
@@ -325,6 +325,11 @@ def group_relabel(group, label):
         echo.echo_critical(str(exception))
     else:
         echo.echo_success(f"Label changed to '{label}'")
+        msg = (
+            'Note that if you are dumping your profile data to disk, to reflect the relabeling of the group, '
+            'run your `verdi profile dump` command again.'
+        )
+        echo.echo_report(msg)
 
 
 @verdi_group.command('description')
@@ -632,3 +637,106 @@ def group_path_ls(path, type_string, recursive, as_table, no_virtual, with_descr
             if no_virtual and child.is_virtual:
                 continue
             echo.echo(child.path, bold=not child.is_virtual)
+
+
+@verdi_group.command('dump')
+@arguments.GROUP()
+@options.PATH()
+@options.DRY_RUN()
+@options.OVERWRITE()
+@options.PAST_DAYS()
+@options.START_DATE()
+@options.END_DATE()
+@options.FILTER_BY_LAST_DUMP_TIME()
+@options.ONLY_TOP_LEVEL_CALCS()
+@options.ONLY_TOP_LEVEL_WORKFLOWS()
+@options.DELETE_MISSING()
+@options.SYMLINK_CALCS()
+@options.INCLUDE_INPUTS()
+@options.INCLUDE_OUTPUTS()
+@options.INCLUDE_ATTRIBUTES()
+@options.INCLUDE_EXTRAS()
+@options.FLAT()
+@options.DUMP_UNSEALED()
+@click.pass_context
+@with_dbenv()
+def group_dump(
+    ctx,
+    group,
+    path,
+    dry_run,
+    overwrite,
+    past_days,
+    start_date,
+    end_date,
+    filter_by_last_dump_time,
+    delete_missing,
+    only_top_level_calcs,
+    only_top_level_workflows,
+    symlink_calcs,
+    include_inputs,
+    include_outputs,
+    include_attributes,
+    include_extras,
+    flat,
+    dump_unsealed,
+):
+    """Dump data of an AiiDA group to disk."""
+
+    import traceback
+    from pathlib import Path
+
+    from aiida.cmdline.utils import echo
+    from aiida.tools._dumping.utils import DumpPaths
+
+    warning_msg = (
+        'This is a new feature which is still in its testing phase. '
+        'If you encounter unexpected behavior or bugs, please report them via Discourse or GitHub.'
+    )
+    echo.echo_warning(warning_msg)
+
+    try:
+        if path is None:
+            group_path = DumpPaths.get_default_dump_path(group)
+            dump_base_output_path = Path.cwd() / group_path
+            echo.echo_report(f'No output path specified. Using default: `{dump_base_output_path}`')
+        else:
+            dump_base_output_path = Path(path).resolve()
+            echo.echo_report(f'Using specified output path: `{dump_base_output_path}`')
+
+        # --- Logical Checks ---
+        if dry_run and overwrite:
+            msg = (
+                '`--dry-run` and `--overwrite` selected (or set in config). Overwrite operation will NOT be performed.'
+            )
+            echo.echo_warning(msg)
+
+        # Run the dumping
+        group.dump(
+            output_path=dump_base_output_path,
+            dry_run=dry_run,
+            overwrite=overwrite,
+            past_days=past_days,
+            start_date=start_date,
+            end_date=end_date,
+            filter_by_last_dump_time=filter_by_last_dump_time,
+            only_top_level_calcs=only_top_level_calcs,
+            only_top_level_workflows=only_top_level_workflows,
+            symlink_calcs=symlink_calcs,
+            include_inputs=include_inputs,
+            include_outputs=include_outputs,
+            include_attributes=include_attributes,
+            include_extras=include_extras,
+            flat=flat,
+            dump_unsealed=dump_unsealed,
+        )
+
+        if not dry_run:
+            msg = f'Raw files for group `{group.label}` dumped into folder `{dump_base_output_path.name}`.'
+            echo.echo_success(msg)
+        else:
+            echo.echo_success('Dry run completed.')
+
+    except Exception as e:
+        msg = f'Unexpected error during dump of group {group.label}:\n ({e!s}).\n'
+        echo.echo_critical(msg + traceback.format_exc())