feat: support fully configured fetches for toolchain and fetch upstreams (#545)

This allows for toolchains for things such as Python wheels to be fetched without being extracted in the wake of #541

Author: bhearsum

src/taskgraph/transforms/run/__init__.py

@@ -14,7 +14,7 @@
 import json
 import logging
 
-from voluptuous import Any, Exclusive, Extra, Optional, Required
+from voluptuous import Exclusive, Extra, Optional, Required
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.cached_tasks import order_tasks
@@ -85,7 +85,6 @@
         },
         # A list of artifacts to install from 'fetch' tasks.
         Optional("fetches"): {
-            Any("toolchain", "fetch"): [str],
             str: [
                 str,
                 fetches_schema,
@@ -252,7 +251,15 @@ def use_fetches(config, tasks):
         for kind in sorted(fetches):
             artifacts = fetches[kind]
             if kind in ("fetch", "toolchain"):
-                for fetch_name in sorted(artifacts):
+                for artifact in sorted(artifacts):
+                    # Convert name only fetch entries to full fledged ones for
+                    # easier processing.
+                    if isinstance(artifact, str):
+                        artifact = {
+                            "artifact": artifact,
+                        }
+
+                    fetch_name = artifact["artifact"]
                     label = f"{kind}-{fetch_name}"
                     label = aliases.get(label, label)
                     if label not in artifact_names:
@@ -263,15 +270,21 @@ def use_fetches(config, tasks):
                         env.update(extra_env[label])
 
                     path = artifact_names[label]
+                    dest = artifact.get("dest", None)
+                    extract = artifact.get("extract", True)
+                    verify_hash = artifact.get("verify-hash", False)
 
                     dependencies[label] = label
-                    task_fetches.append(
-                        {
-                            "artifact": path,
-                            "task": f"<{label}>",
-                            "extract": True,
-                        }
-                    )
+                    fetch = {
+                        "artifact": path,
+                        "task": f"<{label}>",
+                        "extract": extract,
+                    }
+                    if dest is not None:
+                        fetch["dest"] = dest
+                    if verify_hash:
+                        fetch["verify-hash"] = verify_hash
+                    task_fetches.append(fetch)
             else:
                 if kind not in dependencies:
                     raise Exception(

test/test_transforms_run.py

@@ -6,6 +6,7 @@
 Tests for the 'run' transform subsystem.
 """
 
+import json
 import os
 from copy import deepcopy
 from pprint import pprint
@@ -89,6 +90,12 @@ def assert_use_fetches_toolchain_env(task):
     assert task["worker"]["env"]["FOO"] == "1"
 
 
+def assert_use_fetches_toolchain_extract_false(task):
+    fetches = json.loads(task["worker"]["env"]["MOZ_FETCHES"]["task-reference"])
+    assert len(fetches) == 1
+    assert fetches[0]["extract"] is False
+
+
 @pytest.mark.parametrize(
     "task,kind_dependencies_tasks",
     (
@@ -107,6 +114,20 @@ def assert_use_fetches_toolchain_env(task):
             ],
             id="toolchain_env",
         ),
+        pytest.param(
+            {"fetches": {"toolchain": [{"artifact": "foo", "extract": False}]}},
+            [
+                Task(
+                    kind="toolchain",
+                    label="toolchain-foo",
+                    attributes={
+                        "toolchain-artifact": "target.whl",
+                    },
+                    task={},
+                )
+            ],
+            id="toolchain_extract_false",
+        ),
     ),
 )
 def test_use_fetches(