PluginMarket/generate_json.py at main · m8test/PluginMarket · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
import json
import os
import re
import requests
from itertools import islice


def read_file_content(file_path):
    with open(file_path, 'r', encoding='utf-8') as f:
        return f.read().strip()


def validate_url(url, label):
    if url:
        try:
            response = requests.head(url, allow_redirects=True, timeout=10)
            if not response.ok:
                raise Exception(f"[{label}] Invalid URL: {url} - Status code: {response.status_code}")
        except Exception as e:
            raise Exception(f"[{label}] URL validation failed: {url} - {e}")


def check_readme_existence(repository_url, local_readme_path, label):
    def convert_github_url_to_raw_readme(url):
        """
        将 GitHub 仓库 URL 转为 raw 的 README.md 地址
        支持：
        - https://github.com/user/repo
        - https://github.com/user/repo/tree/branch
        """
        pattern_branch = r'https://github\.com/([^/]+)/([^/]+)/tree/([^/]+)$'
        pattern_root = r'https://github\.com/([^/]+)/([^/]+)$'

        match = re.match(pattern_branch, url)
        if match:
            user, repo, branch = match.groups()
            return f'https://raw.githubusercontent.com/{user}/{repo}/{branch}/README.md'

        match = re.match(pattern_root, url)
        if match:
            user, repo = match.groups()
            # 默认使用 main 分支
            return f'https://raw.githubusercontent.com/{user}/{repo}/main/README.md'

        # 不是 GitHub 仓库地址，则保留原方式
        return url.rstrip('/') + '/README.md'

    if repository_url:
        readme_url = convert_github_url_to_raw_readme(repository_url)
        try:
            response = requests.head(readme_url, allow_redirects=True, timeout=10)
            if response.ok:
                return readme_url
            else:
                print(f"[{label}] README.md not found at: {readme_url} - Status: {response.status_code}")
        except Exception as e:
            print(f"[{label}] repository README.md check failed: {readme_url} - {e}")

    if os.path.exists(local_readme_path):
        return "local"

    raise Exception(f"[{label}] Missing README.md (both remote and local)")


def process_directory(base_dir, subdir):
    latest_txt_path = os.path.join(base_dir, 'latest.txt')
    latest_content = read_file_content(latest_txt_path) if os.path.exists(latest_txt_path) else ""

    versions = []
    for item in os.listdir(base_dir):
        item_path = os.path.join(base_dir, item)
        if os.path.isdir(item_path):
            download_url_path = os.path.join(item_path, 'download_url.txt')
            repository_path = os.path.join(item_path, 'repository.txt')
            local_readme_path = os.path.join(item_path, 'README.md')

            download_url = read_file_content(download_url_path) if os.path.exists(download_url_path) else ""
            repository = read_file_content(repository_path) if os.path.exists(repository_path) else ""

            validate_url(download_url, f"{subdir}/{item}/download_url")
            if repository:
                validate_url(repository, f"{subdir}/{item}/repository")

            _ = check_readme_existence(repository, local_readme_path, f"{subdir}/{item}/README.md")

            versions.append({
                "name": item,
                "download_url": download_url,
                "repository": repository
            })

    # ✅ 校验 latest 是否在 versions 中
    version_names = [v["name"] for v in versions]
    if latest_content and latest_content not in version_names:
        raise Exception(f"[{subdir}] latest.txt 的内容 '{latest_content}' 不在 versions 列表中: {version_names}")

    return {
        "latest": latest_content,
        "versions": versions
    }


def chunked_iterable(iterable, size):
    it = iter(iterable)
    for first in it:
        yield [first] + list(islice(it, size - 1))


def main():
    directories = ['common', 'builder', 'component', 'language', 'editor']
    items_per_page = 10

    for dir_name in directories:
        if os.path.exists(dir_name):
            subdirs = [item for item in os.listdir(dir_name) if os.path.isdir(os.path.join(dir_name, item))]
            subdirs.sort()

            output_folder = f'{dir_name}_output'
            os.makedirs(output_folder, exist_ok=True)

            for page_num, subdirs_chunk in enumerate(chunked_iterable(subdirs, items_per_page), start=1):
                page_data = []
                for subdir in subdirs_chunk:
                    subdir_path = os.path.join(dir_name, subdir)
                    data = process_directory(subdir_path, subdir)
                    data["name"] = subdir  # 👈 添加模块名字段
                    page_data.append(data)

                output_file = os.path.join(output_folder, f'page_{page_num}.json')
                with open(output_file, 'w', encoding='utf-8') as f:
                    json.dump(page_data, f, indent=4, ensure_ascii=False)

                print(f"✅ Generated JSON for {dir_name}, page {page_num}: {output_file}")


if __name__ == "__main__":
    main()