diff --git a/trollmoves/filescleaner.py b/trollmoves/filescleaner.py index ce52ca7..88e6d7c 100644 --- a/trollmoves/filescleaner.py +++ b/trollmoves/filescleaner.py @@ -56,20 +56,24 @@ def clean_dir(self, ref_time, pathname_template, **kwargs): section_size = 0 removed = [] - base_path = Path(pathname_template).parent - pattern = Path(pathname_template).name + base_template = str(Path(pathname_template).parent) + file_pattern = Path(pathname_template).name - for dirpath, _dirnames, _ in os.walk(base_path, followlinks=True): - files_in_dir = glob(os.path.join(dirpath, pattern)) + for base_dir in glob(base_template): + if not os.path.isdir(base_dir): + continue - s_size, s_files, removed_files = self.clean_files_and_dirs(files_in_dir, ref_time) - section_files += s_files - section_size += s_size - removed.extend(removed_files) + for dirpath, _dirnames, _ in os.walk(base_dir, followlinks=True): + files_in_dir = glob(os.path.join(dirpath, file_pattern)) + + s_size, s_files, removed_files = self.clean_files_and_dirs(files_in_dir, ref_time) + section_files += s_files + section_size += s_size + removed.extend(removed_files) - # Do not remove the configured/base directory itself - if Path(dirpath) != base_path and not os.listdir(dirpath): - self._remove_empty_directory(dirpath) + # Do not remove the configured/base directory itself + if Path(dirpath) != Path(base_dir) and not os.listdir(dirpath): + self._remove_empty_directory(dirpath) return (section_size, section_files, removed) diff --git a/trollmoves/tests/conftest.py b/trollmoves/tests/conftest.py index 226a988..b680efd 100644 --- a/trollmoves/tests/conftest.py +++ b/trollmoves/tests/conftest.py @@ -86,22 +86,27 @@ def file_structure_with_some_old_files_and_empty_dir(tmp_path): data_dir = tmp_path / "mydata" / "geo_out" data_dir.mkdir(parents=True) - data_subdir1 = data_dir / "imagery" + data_subdir1 = data_dir / "imagery-1" data_subdir1.mkdir(parents=True) - data_subdir2 = data_dir / "empty_dir" + data_subdir2 = data_dir / "imagery-2" data_subdir2.mkdir(parents=True) + data_subdir3 = data_dir / "empty_dir" + data_subdir3.mkdir(parents=True) files = ["a.png", "b.png", "c.tif"] for fname in files: (data_subdir1 / fname).touch() + (data_subdir2 / fname).touch() eight_hours_ago = dt.datetime.now(dt.timezone.utc) - dt.timedelta(hours=8) oldfile = (data_subdir1 / "b.png") os.utime(oldfile, (eight_hours_ago.timestamp(), eight_hours_ago.timestamp())) + oldfile = (data_subdir2 / "c.tif") + os.utime(oldfile, (eight_hours_ago.timestamp(), eight_hours_ago.timestamp())) # Force the directory to be old as well: - os.utime(data_subdir2, (eight_hours_ago.timestamp(), eight_hours_ago.timestamp())) + os.utime(data_subdir3, (eight_hours_ago.timestamp(), eight_hours_ago.timestamp())) - return data_dir, data_subdir1, data_subdir2 + return data_dir, data_subdir1, data_subdir2, data_subdir3 @pytest.fixture diff --git a/trollmoves/tests/test_remove_files.py b/trollmoves/tests/test_remove_files.py index 6a1802c..ff3b637 100644 --- a/trollmoves/tests/test_remove_files.py +++ b/trollmoves/tests/test_remove_files.py @@ -194,7 +194,7 @@ def test_remove_files_path_missing(file_structure_with_some_old_files, caplog): def test_remove_files_empty_dir_mtime(file_structure_with_some_old_files_and_empty_dir, caplog): """Test remove files.""" pub = FakePublisher() - dir_base, sub_dir1, sub_dir2 = file_structure_with_some_old_files_and_empty_dir + dir_base, sub_dir1, _, sub_dir2 = file_structure_with_some_old_files_and_empty_dir basedir = str(dir_base) subdir1 = sub_dir1.name @@ -220,10 +220,38 @@ def test_remove_files_empty_dir_mtime(file_structure_with_some_old_files_and_emp assert not sub_dir2.exists() +def test_remove_files_using_wildcard_in_template_dirs(file_structure_with_some_old_files_and_empty_dir, caplog): + """Test remove files.""" + pub = FakePublisher() + dir_base, sub_dir1, sub_dir2, sub_dir3 = file_structure_with_some_old_files_and_empty_dir + + basedir = str(dir_base) + + section = "mytest_files1" + info = {"mailhost": "localhost", + "to": "some_users@xxx.yy", + "subject": "Cleanup Error on {hostname}", + "base_dir": f"{basedir}", + "stat_time_method": "st_mtime", + "recursive": True, + "templates": f"{dir_base}/imagery-?/*", + "hours": "3"} + + with caplog.at_level(logging.DEBUG): + fcleaner = FilesCleaner(pub, section, info, dry_run=False) + size, num_files, removed_files = fcleaner.clean_section() + + log_output1 = f'Removed {(sub_dir1 / "b.png")}' + assert log_output1 in caplog.text + assert not (sub_dir1 / "b.png").exists() + assert num_files == 2 + assert sub_dir3.exists() + + def test_remove_files_empty_dir_atime(file_structure_with_some_old_files_and_empty_dir, caplog): """Test remove files.""" pub = FakePublisher() - dir_base, sub_dir1, sub_dir2 = file_structure_with_some_old_files_and_empty_dir + dir_base, sub_dir1, _, sub_dir2 = file_structure_with_some_old_files_and_empty_dir basedir = str(dir_base) subdir1 = sub_dir1.name