Handle cases with wildcards inside the path to the basedir-template

trollmoves/filescleaner.py

@@ -56,20 +56,24 @@ def clean_dir(self, ref_time, pathname_template, **kwargs):
         section_size = 0
         removed = []
 
-        base_path = Path(pathname_template).parent
-        pattern = Path(pathname_template).name
+        base_template = str(Path(pathname_template).parent)
+        file_pattern = Path(pathname_template).name
 
-        for dirpath, _dirnames, _ in os.walk(base_path, followlinks=True):
-            files_in_dir = glob(os.path.join(dirpath, pattern))
+        for base_dir in glob(base_template):
+            if not os.path.isdir(base_dir):
+                continue
 
-            s_size, s_files, removed_files = self.clean_files_and_dirs(files_in_dir, ref_time)
-            section_files += s_files
-            section_size += s_size
-            removed.extend(removed_files)
+            for dirpath, _dirnames, _ in os.walk(base_dir, followlinks=True):
+                files_in_dir = glob(os.path.join(dirpath, file_pattern))
+
+                s_size, s_files, removed_files = self.clean_files_and_dirs(files_in_dir, ref_time)
+                section_files += s_files
+                section_size += s_size
+                removed.extend(removed_files)
 
-            # Do not remove the configured/base directory itself
-            if Path(dirpath) != base_path and not os.listdir(dirpath):
-                self._remove_empty_directory(dirpath)
+                # Do not remove the configured/base directory itself
+                if Path(dirpath) != Path(base_dir) and not os.listdir(dirpath):
+                    self._remove_empty_directory(dirpath)
 
         return (section_size, section_files, removed)
 

trollmoves/tests/conftest.py

@@ -86,22 +86,27 @@ def file_structure_with_some_old_files_and_empty_dir(tmp_path):
     data_dir = tmp_path / "mydata" / "geo_out"
     data_dir.mkdir(parents=True)
 
-    data_subdir1 = data_dir / "imagery"
+    data_subdir1 = data_dir / "imagery-1"
     data_subdir1.mkdir(parents=True)
-    data_subdir2 = data_dir / "empty_dir"
+    data_subdir2 = data_dir / "imagery-2"
     data_subdir2.mkdir(parents=True)
+    data_subdir3 = data_dir / "empty_dir"
+    data_subdir3.mkdir(parents=True)
 
     files = ["a.png", "b.png", "c.tif"]
     for fname in files:
         (data_subdir1 / fname).touch()
+        (data_subdir2 / fname).touch()
 
     eight_hours_ago = dt.datetime.now(dt.timezone.utc) - dt.timedelta(hours=8)
     oldfile = (data_subdir1 / "b.png")
     os.utime(oldfile, (eight_hours_ago.timestamp(), eight_hours_ago.timestamp()))
+    oldfile = (data_subdir2 / "c.tif")
+    os.utime(oldfile, (eight_hours_ago.timestamp(), eight_hours_ago.timestamp()))
     # Force the directory to be old as well:
-    os.utime(data_subdir2, (eight_hours_ago.timestamp(), eight_hours_ago.timestamp()))
+    os.utime(data_subdir3, (eight_hours_ago.timestamp(), eight_hours_ago.timestamp()))
 
-    return data_dir, data_subdir1, data_subdir2
+    return data_dir, data_subdir1, data_subdir2, data_subdir3
 
 
 @pytest.fixture

trollmoves/tests/test_remove_files.py

@@ -194,7 +194,7 @@ def test_remove_files_path_missing(file_structure_with_some_old_files, caplog):
 def test_remove_files_empty_dir_mtime(file_structure_with_some_old_files_and_empty_dir, caplog):
     """Test remove files."""
     pub = FakePublisher()
-    dir_base, sub_dir1, sub_dir2 = file_structure_with_some_old_files_and_empty_dir
+    dir_base, sub_dir1, _, sub_dir2 = file_structure_with_some_old_files_and_empty_dir
 
     basedir = str(dir_base)
     subdir1 = sub_dir1.name
@@ -220,10 +220,38 @@ def test_remove_files_empty_dir_mtime(file_structure_with_some_old_files_and_emp
     assert not sub_dir2.exists()
 
 
+def test_remove_files_using_wildcard_in_template_dirs(file_structure_with_some_old_files_and_empty_dir, caplog):
+    """Test remove files."""
+    pub = FakePublisher()
+    dir_base, sub_dir1, sub_dir2, sub_dir3 = file_structure_with_some_old_files_and_empty_dir
+
+    basedir = str(dir_base)
+
+    section = "mytest_files1"
+    info = {"mailhost": "localhost",
+            "to": "some_users@xxx.yy",
+            "subject": "Cleanup Error on {hostname}",
+            "base_dir": f"{basedir}",
+            "stat_time_method": "st_mtime",
+            "recursive": True,
+            "templates": f"{dir_base}/imagery-?/*",
+            "hours": "3"}
+
+    with caplog.at_level(logging.DEBUG):
+        fcleaner = FilesCleaner(pub, section, info, dry_run=False)
+        size, num_files, removed_files = fcleaner.clean_section()
+
+    log_output1 = f'Removed {(sub_dir1 / "b.png")}'
+    assert log_output1 in caplog.text
+    assert not (sub_dir1 / "b.png").exists()
+    assert num_files == 2
+    assert sub_dir3.exists()
+
+
 def test_remove_files_empty_dir_atime(file_structure_with_some_old_files_and_empty_dir, caplog):
     """Test remove files."""
     pub = FakePublisher()
-    dir_base, sub_dir1, sub_dir2 = file_structure_with_some_old_files_and_empty_dir
+    dir_base, sub_dir1, _, sub_dir2 = file_structure_with_some_old_files_and_empty_dir
 
     basedir = str(dir_base)
     subdir1 = sub_dir1.name