diff --git a/setup.py b/setup.py
index beb46d6a7..c167020c7 100644
--- a/setup.py
+++ b/setup.py
@@ -51,7 +51,8 @@
     # minimum 2021.05.0 to have the AbstractArchiveFileSystem
     "fsspec[http]>=2021.05.0",
     # To get datasets from the Datasets Hub on huggingface.co
-    "huggingface-hub>=0.7.0",
+    # minimum 0.10.0 to have huggingface_hub.utils.build_hf_headers
+    "huggingface-hub>=0.10.0",
     # Utilities from PyPA to e.g., compare versions
     "packaging",
 ]
@@ -72,6 +73,7 @@
 TESTS_REQUIRE = [
     # test dependencies
     "absl-py",
+    "cookiecutter",  # for evaluate-cli
     "charcut>=1.1.1",  # for charcut_mt
     "cer>=1.2.0",  # for characTER
     "nltk",  # for NIST and probably others
diff --git a/src/evaluate/commands/evaluate_cli.py b/src/evaluate/commands/evaluate_cli.py
index 80593c4df..86962875d 100644
--- a/src/evaluate/commands/evaluate_cli.py
+++ b/src/evaluate/commands/evaluate_cli.py
@@ -4,7 +4,7 @@
 from pathlib import Path
 
 from cookiecutter.main import cookiecutter
-from huggingface_hub import HfApi, Repository, create_repo
+from huggingface_hub import HfApi, create_repo
 
 from evaluate.utils.logging import get_logger
 
@@ -45,6 +45,18 @@
 """
 
 
+def run_git(args, cwd):
+    subprocess.run(
+        ["git"] + args,
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        check=True,
+        encoding="utf-8",
+        cwd=cwd,
+        env=os.environ.copy(),
+    )
+
+
 def main():
     parser = argparse.ArgumentParser("HuggingFace Evaluate CLI tool", usage="evaluate-cli <command> [<args>]")
     subparsers = parser.add_subparsers()
@@ -94,19 +106,7 @@ def main():
             f"Could not create Space for module at hf.co/spaces/{namespace}/{module_slug}. Make sure this space does not exist already."
         )
         raise exception
-    subprocess.run(
-        f"git clone {repo_url}".split(),
-        stderr=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        check=True,
-        encoding="utf-8",
-        cwd=output_dir,
-        env=os.environ.copy(),
-    )
-
-    repo = Repository(
-        local_dir=output_dir / module_slug,
-    )
+    run_git(["clone", repo_url], cwd=output_dir)
 
     cookiecutter(
         "https://github.com/huggingface/evaluate/",
@@ -117,9 +117,10 @@ def main():
         overwrite_if_exists=True,
     )
 
-    repo.git_add()
-    repo.git_commit("add module default template")
-    repo.git_push()
+    module_dir = output_dir / module_slug
+    run_git(["add", "."], cwd=module_dir)
+    run_git(["commit", "-m", "add module default template"], cwd=module_dir)
+    run_git(["push"], cwd=module_dir)
 
     print(
         INSTRUCTIONS.format(
diff --git a/tests/test_evaluate_cli.py b/tests/test_evaluate_cli.py
new file mode 100644
index 000000000..16717d10a
--- /dev/null
+++ b/tests/test_evaluate_cli.py
@@ -0,0 +1,45 @@
+import importlib
+import sys
+from unittest.mock import patch
+
+import huggingface_hub
+import pytest
+
+
+CLI_MODULE = "evaluate.commands.evaluate_cli"
+
+
+@pytest.fixture
+def cli_without_hub_repository(monkeypatch):
+    """Import the CLI as it would be imported with `huggingface_hub>=1.0.0`.
+
+    `huggingface_hub.Repository` was removed in v1.0.0, so importing it must not be required.
+    """
+    monkeypatch.delattr(huggingface_hub, "Repository", raising=False)
+    monkeypatch.delitem(sys.modules, CLI_MODULE, raising=False)
+    return importlib.import_module(CLI_MODULE)
+
+
+def test_cli_imports_without_hub_repository(cli_without_hub_repository):
+    assert hasattr(cli_without_hub_repository, "main")
+
+
+def test_cli_create_pushes_template_with_git(cli_without_hub_repository, tmp_path):
+    evaluate_cli = cli_without_hub_repository
+    argv = ["evaluate-cli", "create", "Dummy Metric", "--output_dir", str(tmp_path), "--organization", "dummy_org"]
+
+    with patch.object(evaluate_cli, "create_repo") as create_repo, patch.object(
+        evaluate_cli, "cookiecutter"
+    ) as cookiecutter, patch.object(evaluate_cli.subprocess, "run") as subprocess_run, patch.object(sys, "argv", argv):
+        evaluate_cli.main()
+
+    create_repo.assert_called_once()
+    cookiecutter.assert_called_once()
+
+    git_commands = [call.args[0] for call in subprocess_run.call_args_list]
+    assert git_commands == [
+        ["git", "clone", "https://huggingface.co/spaces/dummy_org/dummy_metric"],
+        ["git", "add", "."],
+        ["git", "commit", "-m", "add module default template"],
+        ["git", "push"],
+    ]