analytics: send remote types and git-remote's path hashes (#10816)

Author: skshetry

dvc/analytics.py

@@ -1,10 +1,14 @@
 import json
 import os
+from typing import TYPE_CHECKING, Optional
 
 from dvc.log import logger
 
 from .env import DVC_ANALYTICS_ENDPOINT, DVC_NO_ANALYTICS
 
+if TYPE_CHECKING:
+    from dvc.scm import Base
+
 logger = logger.getChild(__name__)
 
 
@@ -89,20 +93,62 @@ def send(path):
     os.remove(path)
 
 
-def _scm_in_use():
-    from dvc.exceptions import NotDvcRepoError
-    from dvc.repo import Repo
-    from dvc.scm import NoSCM
+def _git_remote_url(scm: Optional["Base"]) -> Optional[str]:
+    from dvc.scm import Git
+
+    if not isinstance(scm, Git):
+        return None
 
-    from .scm import SCM, SCMError
+    from dulwich.porcelain import get_remote_repo
 
+    dulwich_repo = scm.dulwich.repo
     try:
-        scm = SCM(root_dir=Repo.find_root())
-        return type(scm).__name__
-    except SCMError:
-        return NoSCM.__name__
-    except NotDvcRepoError:
-        pass
+        _remote, url = get_remote_repo(dulwich_repo)
+    except IndexError:
+        # IndexError happens when the head is detached
+        _remote, url = get_remote_repo(dulwich_repo, b"origin")
+    # Dulwich returns (None, "origin") if no remote set
+    if (_remote, url) == (None, "origin"):
+        return None
+    return url
+
+
+def _scm_in_use(scm: Optional["Base"]) -> Optional[str]:
+    return type(scm).__name__ if scm else None
+
+
+def _parse_git_remote_path(remote_url: str) -> str:
+    from urllib.parse import urlparse
+
+    from scmrepo.urls import is_scp_style_url
+
+    parsed = urlparse(remote_url)
+    # Windows Path also gets parsed with a drive letter as scheme
+    # https://github.com/python/cpython/issues/86381
+    if parsed.scheme and parsed.scheme in ("http", "https", "git", "ssh"):
+        return parsed.path.strip("/")
+
+    if is_scp_style_url(remote_url):
+        # handle scp-style URL
+        parts = remote_url.split(":", 1)
+        if len(parts) == 2:
+            _, path = parts
+            return path.rstrip("/")
+    return remote_url
+
+
+def _git_remote_path_hash(scm: Optional["Base"]) -> Optional[str]:
+    """Return a hash of the git remote path."""
+    import hashlib
+
+    try:
+        if remote_url := _git_remote_url(scm):
+            path = _parse_git_remote_path(remote_url)
+            h = hashlib.md5(path.encode("utf-8"), usedforsecurity=False)  # for FIPS
+            return h.hexdigest()
+    except Exception:  # noqa: BLE001
+        logger.debug("Failed to get git remote path", exc_info=True)
+    return None
 
 
 def _runtime_info():
@@ -112,6 +158,8 @@ def _runtime_info():
     from iterative_telemetry import _generate_ci_id, find_or_create_user_id
 
     from dvc import __version__
+    from dvc.info import _get_remotes
+    from dvc.repo import Repo
     from dvc.utils import is_binary
 
     ci_id = _generate_ci_id()
@@ -120,13 +168,24 @@ def _runtime_info():
     else:
         group_id, user_id = None, find_or_create_user_id()
 
+    scm = None
+    remotes = None
+    try:
+        repo = Repo()
+        scm = repo.scm
+        remotes = _get_remotes(repo.config)
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("failed to open repo: %s", exc)
+
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
-        "scm_class": _scm_in_use(),
+        "scm_class": _scm_in_use(scm),
         "system_info": _system_info(),
         "user_id": user_id,
         "group_id": group_id,
+        "remotes": remotes,
+        "git_remote_hash": _git_remote_path_hash(scm),
     }
 
 

dvc/info.py

@@ -48,7 +48,10 @@ def get_dvc_info():
                 info.append("Cache types: " + error_link("no-dvc-cache"))
 
             info.append(f"Caches: {_get_caches(repo.cache)}")
-            info.append(f"Remotes: {_get_remotes(repo.config)}")
+
+            configured_remotes = _get_remotes(repo.config)
+            remotes = ", ".join(configured_remotes) if configured_remotes else None
+            info.append(f"Remotes: {remotes}")
 
             root_directory = repo.root_dir
             fs_root = _get_fs_type(os.path.abspath(root_directory))
@@ -75,12 +78,10 @@ def _get_caches(cache):
 
 
 def _get_remotes(config):
-    schemes = (
+    return [
         get_fs_cls(get_fs_config(config, name=remote)).protocol
         for remote in config["remote"]
-    )
-
-    return ", ".join(schemes) or "None"
+    ]
 
 
 def _get_linktype_support_info(repo):

tests/func/test_analytics.py

@@ -2,6 +2,7 @@
 
 import pytest
 
+from dvc import __version__, env
 from dvc.analytics import _scm_in_use, collect_and_send_report
 from dvc.cli import main
 from dvc.repo import Repo
@@ -33,27 +34,37 @@ def func(argv):
     return mocker.patch("dvc.daemon.daemon", mocker.MagicMock(side_effect=func))
 
 
-def test_collect_and_send_report(mocker, dvc, mock_daemon):
+def test_collect_and_send_report(monkeypatch, mocker, dvc, mock_daemon):
+    monkeypatch.delenv(env.DVC_ANALYTICS_ENDPOINT, raising=False)
     mock_post = mocker.patch("requests.post")
     collect_and_send_report()
 
     assert mock_daemon.call_count == 1
     assert mock_post.call_count == 1
     assert mock_post.call_args == mocker.call(
         "https://analytics.dvc.org",
-        json=ANY(dict),
+        json={
+            "dvc_version": __version__,
+            "scm_class": type(dvc.scm).__name__,
+            "is_binary": False,
+            "system_info": ANY(dict),
+            "user_id": ANY(str),
+            "group_id": mocker.ANY,
+            "remotes": ANY(list),
+            "git_remote_hash": None,
+        },
         headers={"content-type": "application/json"},
         timeout=5,
     )
 
 
 def test_scm_dvc_only(tmp_dir, dvc):
-    scm = _scm_in_use()
+    scm = _scm_in_use(dvc.scm)
     assert scm == "NoSCM"
 
 
 def test_scm_git(tmp_dir, scm, dvc):
-    scm = _scm_in_use()
+    scm = _scm_in_use(scm)
     assert scm == "Git"
 
 
@@ -62,7 +73,7 @@ def test_scm_subrepo(tmp_dir, scm):
     subdir.mkdir()
 
     with subdir.chdir():
-        Repo.init(subdir=True)
-        scm = _scm_in_use()
+        repo = Repo.init(subdir=True)
+        scm = _scm_in_use(repo.scm)
 
     assert scm == "Git"

tests/unit/test_analytics.py

@@ -1,10 +1,11 @@
+import hashlib
 import json
 import platform
 
 import pytest
 from voluptuous import Any, Schema
 
-from dvc import analytics
+from dvc import analytics, env
 from dvc.cli import parse_args
 
 
@@ -50,14 +51,17 @@ def test_runtime_info(tmp_global_dir):
             "user_id": str,
             "system_info": dict,
             "group_id": Any(str, None),
+            "remotes": Any(list, None),
+            "git_remote_hash": Any(str, None),
         },
         required=True,
     )
 
     assert schema(analytics._runtime_info())
 
 
-def test_send(mocker, tmp_path):
+def test_send(monkeypatch, mocker, tmp_path):
+    monkeypatch.delenv(env.DVC_ANALYTICS_ENDPOINT, raising=False)
     mock_post = mocker.patch("requests.post")
 
     import requests
@@ -158,3 +162,38 @@ def test_system_info():
         )
 
     assert schema(analytics._system_info())
+
+
+@pytest.mark.parametrize(
+    "git_remote",
+    [
+        "git://github.com/iterative/dvc.git",
+        "[email protected]:iterative/dvc.git",
+        "http://github.com/iterative/dvc.git",
+        "https://github.com/iterative/dvc.git",
+        "ssh://[email protected]/iterative/dvc.git",
+    ],
+)
+def test_git_remote_hash(mocker, git_remote):
+    m = mocker.patch("dvc.analytics._git_remote_url", return_value=git_remote)
+    expected = hashlib.md5(b"iterative/dvc.git").hexdigest()
+
+    assert analytics._git_remote_path_hash(None) == expected
+    m.assert_called_once_with(None)
+
+
+@pytest.mark.parametrize(
+    "git_remote",
+    [
+        "C:\\Users\\user\\dvc.git",
+        "/home/user/dvc.git",
+        "file:///home/user/dvc.git",
+        "./dvc.git",
+    ],
+)
+def test_git_remote_hash_local(mocker, git_remote):
+    m = mocker.patch("dvc.analytics._git_remote_url", return_value=git_remote)
+
+    expected = hashlib.md5(git_remote.encode("utf-8")).hexdigest()
+    assert analytics._git_remote_path_hash(None) == expected
+    m.assert_called_once_with(None)