From c0b9dac1f5b013cdeeb21328d4251d0290890142 Mon Sep 17 00:00:00 2001 From: Benjamin Milan Date: Wed, 15 Jul 2026 10:54:57 +0100 Subject: [PATCH 1/3] BWDO-809 Implement clone cache lock --- setup.py | 1 + src/sc/clone/cloners/repo_cloner.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7d0526d..8693e81 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ def read_version(): 'pyyaml~=6.0', 'rich>=14', 'requests==2.31.0', # Docker SDK breaks on 2.32.0 + 'filelock==3.29.7', 'repo_library @ git+https://github.com/rdkcentral/sc-repo-library.git@master', 'git_flow_library @ git+https://github.com/rdkcentral/sc-git-flow-library.git@master', 'sc_manifest_parser @ git+https://github.com/rdkcentral/sc-manifest-parser.git@main' diff --git a/src/sc/clone/cloners/repo_cloner.py b/src/sc/clone/cloners/repo_cloner.py index f531212..c4e1089 100644 --- a/src/sc/clone/cloners/repo_cloner.py +++ b/src/sc/clone/cloners/repo_cloner.py @@ -19,6 +19,7 @@ import subprocess import sys +from filelock import FileLock, Timeout from pydantic import BaseModel from .cloner import Cloner, RefType @@ -29,6 +30,7 @@ logger = logging.getLogger(__name__) REPO_CACHE_DIR = Path.home() / ".caches" +LOCK_FILE_PATH = REPO_CACHE_DIR / ".lock" class RepoClonerConfig(BaseModel): """ @@ -74,8 +76,19 @@ def clone(self, directory: Path): - Parses the manifest to retrieve projects. - Initializes GitFlow for all unlocked projects. """ - reference = self._cache() if self.config.cache else None + if self.config.cache: + REPO_CACHE_DIR.mkdir(exist_ok=True) + try: + with FileLock(LOCK_FILE_PATH, timeout=600): + reference = self._cache() + self._clone(directory, reference) + except Timeout: + logger.error(f"The cache lock file {LOCK_FILE_PATH} remained 10 minutes.") + sys.exit(1) + else: + self._clone(directory, None) + def _clone(self, directory: Path, reference: Path | None): self._init_repo(directory=directory, reference=reference) RepoLibrary.sync( directory, @@ -95,7 +108,6 @@ def _cache(self) -> Path: Returns: Path: The directory of the mirrored cache. """ - REPO_CACHE_DIR.mkdir(exist_ok=True) manifest_hostname = self._get_manifest_hostname(self.config.uri) host_cache_dir = Path(REPO_CACHE_DIR / manifest_hostname) host_cache_dir.mkdir(exist_ok=True) From 9310378d554fbf1761a32d908aa7e637452cbe25 Mon Sep 17 00:00:00 2001 From: Benjamin Milan Date: Wed, 15 Jul 2026 11:46:37 +0100 Subject: [PATCH 2/3] BWDO-809 Implement cache lock --- src/sc/clone/cloners/repo_cloner.py | 43 ++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/sc/clone/cloners/repo_cloner.py b/src/sc/clone/cloners/repo_cloner.py index c4e1089..f92c4b6 100644 --- a/src/sc/clone/cloners/repo_cloner.py +++ b/src/sc/clone/cloners/repo_cloner.py @@ -18,6 +18,7 @@ import shutil import subprocess import sys +import time from filelock import FileLock, Timeout from pydantic import BaseModel @@ -30,7 +31,8 @@ logger = logging.getLogger(__name__) REPO_CACHE_DIR = Path.home() / ".caches" -LOCK_FILE_PATH = REPO_CACHE_DIR / ".lock" +CACHE_LOCK_PATH = REPO_CACHE_DIR / ".lock" +CACHE_MAX_WAIT = 600 class RepoClonerConfig(BaseModel): """ @@ -77,18 +79,39 @@ def clone(self, directory: Path): - Initializes GitFlow for all unlocked projects. """ if self.config.cache: - REPO_CACHE_DIR.mkdir(exist_ok=True) + self._clone_with_cache(directory) + else: + self._clone(directory) + + def _clone_with_cache(self, directory: Path): + REPO_CACHE_DIR.mkdir(exist_ok=True) + + started = time.monotonic() + lock = FileLock(CACHE_LOCK_PATH) + + while True: try: - with FileLock(LOCK_FILE_PATH, timeout=600): - reference = self._cache() - self._clone(directory, reference) + lock.acquire(timeout=5) + break except Timeout: - logger.error(f"The cache lock file {LOCK_FILE_PATH} remained 10 minutes.") - sys.exit(1) - else: - self._clone(directory, None) + waited = int(time.monotonic() - started) + if waited < CACHE_MAX_WAIT: + logger.info( + f"Cache is in use by another process, waited {waited} seconds..." + ) + else: + logger.error( + f"Cache remained lock past the set wait time of {CACHE_MAX_WAIT} seconds." + ) + sys.exit(1) + + try: + reference = self._cache() + self._clone(directory, reference) + finally: + lock.release() - def _clone(self, directory: Path, reference: Path | None): + def _clone(self, directory: Path, reference: Path | None = None): self._init_repo(directory=directory, reference=reference) RepoLibrary.sync( directory, From f56a870d7866907590d5a92dec6fa52b2293e5ec Mon Sep 17 00:00:00 2001 From: Benjamin Milan Date: Wed, 15 Jul 2026 13:18:09 +0100 Subject: [PATCH 3/3] BWDO-809 increased timeout to 20s --- src/sc/clone/cloners/repo_cloner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sc/clone/cloners/repo_cloner.py b/src/sc/clone/cloners/repo_cloner.py index f92c4b6..e3577bc 100644 --- a/src/sc/clone/cloners/repo_cloner.py +++ b/src/sc/clone/cloners/repo_cloner.py @@ -89,9 +89,10 @@ def _clone_with_cache(self, directory: Path): started = time.monotonic() lock = FileLock(CACHE_LOCK_PATH) + logger.info("Acquiring cache lock.") while True: try: - lock.acquire(timeout=5) + lock.acquire(timeout=20) break except Timeout: waited = int(time.monotonic() - started) @@ -101,9 +102,10 @@ def _clone_with_cache(self, directory: Path): ) else: logger.error( - f"Cache remained lock past the set wait time of {CACHE_MAX_WAIT} seconds." + f"Cache remained locked past the set wait time of {CACHE_MAX_WAIT} seconds." ) sys.exit(1) + logger.info("Cache lock acquired.") try: reference = self._cache()