Prepare Makefile to make CTSM5.4 fsurdat/landuse datasets

.gitignore

@@ -7,7 +7,11 @@
 # editor files
 *.swp
 *~
+
+# MS VS Code
 .vscode/
+# but don't ignore the main .vscode directory here:
+!/.vscode
 
 # vim files (from https://github.com/github/gitignore/blob/master/Global/Vim.gitignore)
 # Swap

.gitmodules

@@ -28,15 +28,15 @@
 [submodule "fates"]
 path = src/fates
 url = https://github.com/NGEET/fates
-fxtag = sci.1.87.0_api.41.0.0
+fxtag = sci.1.87.2_api.41.0.0
 fxrequired = AlwaysRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/NGEET/fates
 
 [submodule "cism"]
 path = components/cism
 url = https://github.com/ESCOMP/CISM-wrapper
-fxtag = cismwrap_2_2_006
+fxtag = cismwrap_2_2_010
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CISM-wrapper
@@ -68,31 +68,31 @@ fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute
 [submodule "ccs_config"]
 path = ccs_config
 url = https://github.com/ESMCI/ccs_config_cesm.git
-fxtag = ccs_config_cesm1.0.56
+fxtag = ccs_config_cesm1.0.61
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 
 [submodule "cime"]
 path = cime
 url = https://github.com/ESMCI/cime
-fxtag = cime6.1.113
+fxtag = cime6.1.128
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/cime
 
 [submodule "cmeps"]
 path = components/cmeps
 url = https://github.com/ESCOMP/CMEPS.git
-fxtag = cmeps1.1.5
+fxtag = cmeps1.1.20
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
 
 [submodule "cdeps"]
 path = components/cdeps
 url = https://github.com/ESCOMP/CDEPS.git
-fxtag = cdeps1.0.79
+fxtag = cdeps1.0.81
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git

.lib/git-fleximod/git_fleximod/cli.py

@@ -1,38 +1,50 @@
 from pathlib import Path
-import argparse
+import argparse, os, sys
 from git_fleximod import utils
 
-__version__ = "0.9.3"
+__version__ = "1.0.2"
+
+class CustomArgumentParser(argparse.ArgumentParser):
+    def print_help(self, file=None):
+        # First print the default help message
+        super().print_help(file)
+
+        # Then append the contents of README.md
+        candidate_paths = [
+            Path(sys.prefix) / "share" / "git_fleximod" / "README.md",
+            Path(__file__).resolve().parent.parent / "README.md",  # fallback for dev
+        ]
+        for path in candidate_paths:
+            if os.path.exists(path):
+                with open(path) as f:
+                    print( f.read(), file=file)
+                    return
+        print( "README.md not found.", file=file)
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree
     which contains a file called filename """
-    try:
-        root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"],
-                                        output_to_caller=True ).rstrip()
-    except:
-        d = Path.cwd()
-        root = Path(d.root)
-        dirlist = []
-        dl = d
-        while dl != root:
-            dirlist.append(dl)
-            dl = dl.parent
-        dirlist.append(root)
-        dirlist.reverse()
-
-        for dl in dirlist:
-            attempt = dl / filename
-            if attempt.is_file():
-                return str(dl)
-        return None
-    return Path(root)
+    d = Path.cwd()
+    root = Path(d.root)
+    dirlist = []
+    dl = d
+    while dl != root:
+        dirlist.append(dl)
+        dl = dl.parent
+    dirlist.append(root)
+    dirlist.reverse()
+
+    for dl in dirlist:
+        attempt = dl / filename
+        if attempt.is_file():
+            return str(dl)
+    return None
 
 def get_parser():
     description = """
     %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models
     """
-    parser = argparse.ArgumentParser(
+    parser = CustomArgumentParser(
         description=description, formatter_class=argparse.RawDescriptionHelpFormatter
     )
 

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -9,6 +9,7 @@
 import shutil
 import logging
 import textwrap
+import asyncio
 from git_fleximod import utils
 from git_fleximod import cli
 from git_fleximod.gitinterface import GitInterface
@@ -181,6 +182,8 @@ def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger):
     url = gitmodules.get(name, "url")
     assert path and url, f"Malformed .gitmodules file {path} {url}"
     tag = gitmodules.get(name, "fxtag")
+    if not tag:
+        tag = gitmodules.get(name, "hash")
     fxurl = gitmodules.get(name, "fxDONOTUSEurl")
     fxsparse = gitmodules.get(name, "fxsparse")
     fxrequired = gitmodules.get(name, "fxrequired")
@@ -216,10 +219,10 @@ def git_toplevelroot(root_dir, logger):
     _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
     return superroot
 
-def submodules_update(gitmodules, root_dir, requiredlist, force):
-    for name in gitmodules.sections():
+async def submodules_update(gitmodules, root_dir, requiredlist, force):
+    async def update_submodule(name, requiredlist, force):
         submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger)
-    
+
         _, needsupdate, localmods, testfails = submod.status()
         if not submod.fxrequired:
             submod.fxrequired = "AlwaysRequired"
@@ -237,11 +240,11 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
             if "Optional" in fxrequired and "Optional" not in requiredlist:
                 if fxrequired.startswith("Always"):
                     print(f"Skipping optional component {name:>20}")
-                continue
+                return # continue to next submodule
         optional = "AlwaysOptional" in requiredlist
 
         if fxrequired in requiredlist:
-            submod.update()
+            await submod.update()
             repodir = os.path.join(root_dir, submod.path)
             if os.path.exists(os.path.join(repodir, ".gitmodules")):
                 # recursively handle this checkout
@@ -250,8 +253,10 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 newrequiredlist = ["AlwaysRequired"]
                 if optional:
                     newrequiredlist.append("AlwaysOptional")
+                await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
 
-                submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
+    tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()]
+    await asyncio.gather(*tasks)
 
 def local_mods_output():
     text = """\
@@ -345,7 +350,7 @@ def main():
         sys.exit(f"No submodule components found, root_dir={root_dir}")
     retval = 0
     if action == "update":
-        submodules_update(gitmodules, root_dir, fxrequired, force)
+        asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force))
     elif action == "status":
         tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True)
         if tfails + lmods + updates > 0:

.lib/git-fleximod/git_fleximod/gitinterface.py

@@ -2,6 +2,7 @@
 import sys
 from . import utils
 from pathlib import Path
+import asyncio
 
 class GitInterface:
     def __init__(self, repo_path, logger):
@@ -47,6 +48,16 @@ def _init_git_repo(self):
             command = ("git", "-C", str(self.repo_path), "init")
             utils.execute_subprocess(command)
 
+    def _git_operation_command(self, operation, args):
+        newargs = []
+        for a in args:
+            # Do not use ssh interface
+            if isinstance(a, str):
+                a = a.replace("[email protected]:", "https://github.com/")
+            newargs.append(a)
+
+        return self._git_command(operation, *newargs)
+
     # pylint: disable=unused-argument
     def git_operation(self, operation, *args, **kwargs):
         newargs = []
@@ -66,6 +77,25 @@ def git_operation(self, operation, *args, **kwargs):
         else:
             return 0, command
 
+    # pylint: disable=unused-argument
+    async def git_operation_async(self, operation, *args, **kwargs):
+        command = self._git_operation_command(operation, args)
+        if isinstance(command, list):
+            try:
+                process = await asyncio.create_subprocess_exec(
+                    *command,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE
+                )
+                stdout, stderr = await process.communicate()
+                status = process.returncode
+                output = stdout.decode().strip() if stdout else stderr.decode().strip()
+                return status, output
+            except Exception as e:
+                sys.exit(e)
+        else:
+            return 0, command
+
     def config_get_value(self, section, name):
         if self._use_module:
             config = self.repo.config_reader()