From fdc9b1658f19531b96a932b257a4fc36dc678090 Mon Sep 17 00:00:00 2001 From: SNOWNUMB Date: Wed, 4 Feb 2026 17:53:44 +0500 Subject: [PATCH 1/3] docs: update roadmap and tasks in README.md --- README.md | 57 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 937be1f..8d3c9be 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ **StackScout** — это интеллектуальный микросервис на **Java Spring Boot** для **управления программными активами (Software Asset Management)**, который автоматизирует сбор, анализ и мониторинг open-source библиотек из PyPI и Docker Hub. Наша платформа предоставляет: + - Оценку "здоровья" библиотек в реальном времени - Валидацию соответствия лицензиям - Автоматический сбор метаданных @@ -61,25 +62,27 @@ ### Основные компоненты -| Компонент | Технология | Назначение | -|-----------|-----------|-----------| -| **API слой** | Spring Boot + Spring Web | RESTful endpoints с JWT авторизацией | -| **Слой данных** | PostgreSQL + Spring Data JPA | Постоянное хранилище и ORM | -| **Слой кэширования** | Redis + Spring Cache | Управление сессиями и ограничение запросов | -| **Система очередей** | Spring AMQP + RabbitMQ | Асинхронный парсинг и планирование | -| **Мониторинг** | Prometheus + Grafana | Сбор метрик и визуализация | +| Компонент | Технология | Назначение | +| -------------------- | ---------------------------- | ------------------------------------------ | +| **API слой** | Spring Boot + Spring Web | RESTful endpoints с JWT авторизацией | +| **Слой данных** | PostgreSQL + Spring Data JPA | Постоянное хранилище и ORM | +| **Слой кэширования** | Redis + Spring Cache | Управление сессиями и ограничение запросов | +| **Система очередей** | Spring AMQP + RabbitMQ | Асинхронный парсинг и планирование | +| **Мониторинг** | Prometheus + Grafana | Сбор метрик и визуализация | --- ## Возможности ### Сборщик данных (Data Collector) + - Автоматический сбор метаданных через официальные API (PyPI JSON API, Docker Registry API) - Планируемый парсинг через Cron jobs (настраиваемые интервалы) - Параллельная обработка с использованием очередей задач - Нормализация и дедупликация данных ### Модуль лицензионного соответствия + - Нормализация названий лицензий (`"MIT License"` → `"MIT"`) - Классификация рисков: - **Разрешающие**: MIT, Apache-2.0, BSD @@ -88,19 +91,23 @@ - Матрица совместимости для проектов с множественными лицензиями ### Алгоритм оценки здоровья + Интеллектуальная система оценки на основе: + - **Актуальность**: Время с момента последнего релиза (0-40 баллов) - **Активность**: Количество релизов за последний год (0-30 баллов) - **Репозиторий**: Наличие официального исходного кода (0-20 баллов) - **Сообщество**: Количество контрибьюторов и активность (0-10 баллов) **Интерпретация оценок**: + - **80-100**: Отлично (Готово к production) - **60-79**: Хорошо (Рекомендуется проверка) - **40-59**: Удовлетворительно (Использовать с осторожностью) - **0-39**: Плохо (Избегать или заменить) ### Защищенный API + - JWT-авторизация (Spring Security) - Управление доступом на основе ролей (RBAC) - Ограничение количества запросов (100 запросов/мин на API ключ) @@ -108,6 +115,7 @@ - Спецификация OpenAPI 3.0 (SpringDoc) ### Наблюдаемость (Observability) + - Пользовательские метрики Prometheus (`http_requests_total`, `library_scan_duration`) - Готовые дашборды Grafana - Spring Boot Actuator для проверки здоровья (`/actuator/health`, `/actuator/prometheus`) @@ -127,16 +135,18 @@ Проект настроен для работы "из коробки" без дополнительных настроек! 1. **Клонируйте репозиторий** + ```bash git clone https://github.com/NeonByte-Education/StackScout.git cd StackScout ``` 2. **Запустите приложение** + ```bash ./gradlew bootRun ``` - + Приложение автоматически: - Использует встроенную H2 базу данных в памяти - Запустится на порту 8081 @@ -157,16 +167,19 @@ Для полноценного окружения с PostgreSQL, Redis и RabbitMQ: 1. **Запустите сервисы через Docker Compose** + ```bash docker-compose up -d ``` 2. **Запустите приложение с production профилем** + ```bash ./gradlew bootRun --args='--spring.profiles.active=prod' ``` Или без профиля (по умолчанию использует PostgreSQL): + ```bash ./gradlew bootRun ``` @@ -174,6 +187,7 @@ ### Режимы работы #### Dev режим (по умолчанию) + - H2 база данных в памяти - Без Redis и RabbitMQ - H2 Console включена @@ -185,6 +199,7 @@ ``` #### Production режим + - PostgreSQL база данных - Redis для кэширования - RabbitMQ для очередей @@ -226,18 +241,21 @@ docker-compose up -d ### Примеры endpoints #### Поиск библиотек + ```http GET /api/v1/libraries/search?query=nestjs&source=npm Authorization: Bearer ``` #### Получение деталей библиотеки + ```http GET /api/v1/libraries/:id Authorization: Bearer ``` Ответ: + ```json { "id": "uuid-here", @@ -252,6 +270,7 @@ Authorization: Bearer ``` #### Запуск ручного сканирования + ```http POST /api/v1/collector/scan Authorization: Bearer @@ -268,37 +287,40 @@ Content-Type: application/json ## Roadmap ### Неделя 1-3: Фундамент + - [x] Настройка проекта с Spring Boot - [x] Конфигурация Docker окружения - [x] Проектирование схемы БД (Spring Data JPA) -- [ ] Базовые CRUD операции +- [x] Базовые CRUD операции ### Неделя 4-6: Основные функции -- [ ] Интеграция с PyPI API -- [ ] Интеграция с Docker Hub API -- [ ] Модуль нормализации лицензий -- [ ] Калькулятор оценки здоровья + +- [x] Интеграция с PyPI API +- [x] Интеграция с Docker Hub API +- [x] Модуль нормализации лицензий +- [x] Калькулятор оценки здоровья ### Неделя 7-9: Продвинутые функции + - [ ] Система JWT-авторизации (Spring Security) - [ ] Настройка очереди задач RabbitMQ - [ ] Планировщик сборщика (@Scheduled) - [ ] Слой кэширования Redis (Spring Cache) ### Неделя 10-12: Качество & DevOps + - [ ] Unit & E2E тестирование (Jest) - [ ] CI/CD пайплайн GitHub Actions - [ ] Интеграция метрик Prometheus - [ ] Дашборды Grafana ### Неделя 13-15: Готовность к production + - [ ] Ограничение частоты запросов API - [ ] Полная документация - [ ] Оптимизация производительности - [ ] Аудит безопасности - - --- ## Технологический стек @@ -308,6 +330,7 @@ Content-Type: application/json **Backend** + - Java 17 - Spring Boot 3.2 - Spring Security @@ -317,6 +340,7 @@ Content-Type: application/json **База данных** + - PostgreSQL - Flyway Migrations - Redis @@ -326,6 +350,7 @@ Content-Type: application/json **DevOps** + - Docker - GitHub Actions - Prometheus + Grafana @@ -348,6 +373,7 @@ Content-Type: application/json 5. Откройте Pull Request Пожалуйста, убедитесь, что: + - Все тесты проходят (`npm run test`) - Код соответствует правилам ESLint (`npm run lint`) - Сообщения коммитов следуют [Conventional Commits](https://www.conventionalcommits.org/) @@ -366,6 +392,7 @@ Content-Type: application/json Проект по Advanced Backend & DevOps **Контакты:** + - Email: contact@stackscout.dev - GitHub: [@S-NOWNUM-B](https://github.com/S-NOWNUM-B) - GitHub: [@LINESKL](https://github.com/LINESKL) From 232c813f0fbb3f8dd3ae929a4743122a0306a7a6 Mon Sep 17 00:00:00 2001 From: SNOWNUMB Date: Wed, 4 Feb 2026 17:55:08 +0500 Subject: [PATCH 2/3] feat: reconfigure postgres to standard port 5432 --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ba1512d..8e28d6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.8' +version: "3.8" services: postgres: @@ -51,4 +51,4 @@ services: volumes: postgres_data: redis_data: - rabbitmq_data: \ No newline at end of file + rabbitmq_data: From 7396355b33bdae4388966e9ed6e74fcb0afb7762 Mon Sep 17 00:00:00 2001 From: SNOWNUMB Date: Wed, 4 Feb 2026 18:13:21 +0500 Subject: [PATCH 3/3] Add file comments to all source files --- .../com/stackscout/StackScoutApplication.java | 1 + .../com/stackscout/config/CorsConfig.java | 1 + .../com/stackscout/config/OpenApiConfig.java | 1 + .../com/stackscout/config/RabbitMQConfig.java | 1 + .../com/stackscout/config/RedisConfig.java | 1 + .../com/stackscout/config/SecurityConfig.java | 1 + .../controller/CollectorController.java | 1 + .../controller/HealthCheckController.java | 1 + .../controller/LibraryController.java | 1 + .../controller/LicenseController.java | 1 + .../controller/ScanJobController.java | 1 + .../stackscout/dto/CreateLibraryRequest.java | 1 + .../stackscout/dto/CreateLicenseRequest.java | 1 + .../stackscout/dto/CreateScanJobRequest.java | 1 + .../com/stackscout/dto/ErrorResponse.java | 1 + .../java/com/stackscout/dto/LibraryDto.java | 1 + .../stackscout/dto/LibrarySearchRequest.java | 1 + .../java/com/stackscout/dto/LicenseDto.java | 1 + .../java/com/stackscout/dto/ScanJobDto.java | 1 + .../java/com/stackscout/dto/ScanRequest.java | 1 + .../stackscout/dto/UpdateLibraryRequest.java | 1 + .../dto/dockerhub/DockerHubDTOs.java | 19 ++++ .../com/stackscout/dto/pypi/PyPiDTOs.java | 27 +++++ .../exception/BadRequestException.java | 1 + .../exception/GlobalExceptionHandler.java | 1 + .../exception/ResourceNotFoundException.java | 1 + .../exception/UnauthorizedException.java | 1 + .../com/stackscout/mapper/LibraryMapper.java | 1 + .../java/com/stackscout/model/Library.java | 1 + .../java/com/stackscout/model/License.java | 1 + src/main/java/com/stackscout/model/Role.java | 1 + .../java/com/stackscout/model/ScanJob.java | 1 + .../repository/LibraryRepository.java | 1 + .../repository/LicenseRepository.java | 1 + .../repository/ScanJobRepository.java | 1 + .../scheduler/CollectorScheduler.java | 1 + .../stackscout/service/CollectorService.java | 27 ++++- .../stackscout/service/DockerHubService.java | 18 +++- .../service/HealthScoreService.java | 17 ++- .../stackscout/service/LibraryService.java | 1 + .../stackscout/service/LicenseService.java | 22 ++-- .../com/stackscout/service/PyPiService.java | 17 ++- .../stackscout/service/ScanJobService.java | 1 + .../service/impl/CollectorServiceImpl.java | 96 ++++++++++++++++- .../service/impl/DockerHubServiceImpl.java | 73 ++++++++++++- .../service/impl/HealthScoreServiceImpl.java | 74 +++++++++++++ .../service/impl/LibraryServiceImpl.java | 1 + .../service/impl/LicenseServiceImpl.java | 85 +++++++++------ .../service/impl/PyPiServiceImpl.java | 86 ++++++++++++++- .../service/impl/ScanJobServiceImpl.java | 1 + .../java/com/stackscout/util/DateUtil.java | 1 + .../com/stackscout/util/ValidationUtil.java | 1 + src/main/resources/application.yml | 21 ++-- .../db/migration/V1__init_schema.sql | 17 +++ .../db/migration/V2__add_licenses.sql | 14 +++ .../db/migration/V3__add_scan_jobs.sql | 17 +++ .../StackScoutApplicationTests.java | 1 + .../impl/DockerHubServiceImplTest.java | 102 ++++++++++++++++++ .../impl/HealthScoreServiceImplTest.java | 51 +++++++++ .../service/impl/LicenseServiceImplTest.java | 42 ++++++++ .../service/impl/PyPiServiceImplTest.java | 84 +++++++++++++++ src/test/resources/application-test.yml | 0 62 files changed, 886 insertions(+), 64 deletions(-) create mode 100644 src/main/java/com/stackscout/dto/dockerhub/DockerHubDTOs.java create mode 100644 src/main/java/com/stackscout/dto/pypi/PyPiDTOs.java create mode 100644 src/test/java/com/stackscout/service/impl/DockerHubServiceImplTest.java create mode 100644 src/test/java/com/stackscout/service/impl/HealthScoreServiceImplTest.java create mode 100644 src/test/java/com/stackscout/service/impl/LicenseServiceImplTest.java create mode 100644 src/test/java/com/stackscout/service/impl/PyPiServiceImplTest.java delete mode 100644 src/test/resources/application-test.yml diff --git a/src/main/java/com/stackscout/StackScoutApplication.java b/src/main/java/com/stackscout/StackScoutApplication.java index 69646de..30c6a58 100644 --- a/src/main/java/com/stackscout/StackScoutApplication.java +++ b/src/main/java/com/stackscout/StackScoutApplication.java @@ -1,3 +1,4 @@ +// File: StackScoutApplication.java package com.stackscout; import org.springframework.boot.SpringApplication; diff --git a/src/main/java/com/stackscout/config/CorsConfig.java b/src/main/java/com/stackscout/config/CorsConfig.java index e69de29..3373d10 100644 --- a/src/main/java/com/stackscout/config/CorsConfig.java +++ b/src/main/java/com/stackscout/config/CorsConfig.java @@ -0,0 +1 @@ +// File: CorsConfig.java diff --git a/src/main/java/com/stackscout/config/OpenApiConfig.java b/src/main/java/com/stackscout/config/OpenApiConfig.java index e69de29..16bee2a 100644 --- a/src/main/java/com/stackscout/config/OpenApiConfig.java +++ b/src/main/java/com/stackscout/config/OpenApiConfig.java @@ -0,0 +1 @@ +// File: OpenApiConfig.java diff --git a/src/main/java/com/stackscout/config/RabbitMQConfig.java b/src/main/java/com/stackscout/config/RabbitMQConfig.java index 5c70c59..21ac8d4 100644 --- a/src/main/java/com/stackscout/config/RabbitMQConfig.java +++ b/src/main/java/com/stackscout/config/RabbitMQConfig.java @@ -1,3 +1,4 @@ +// File: RabbitMQConfig.java package com.stackscout.config; public class RabbitMQConfig { diff --git a/src/main/java/com/stackscout/config/RedisConfig.java b/src/main/java/com/stackscout/config/RedisConfig.java index 5d461f7..52fd515 100644 --- a/src/main/java/com/stackscout/config/RedisConfig.java +++ b/src/main/java/com/stackscout/config/RedisConfig.java @@ -1,3 +1,4 @@ +// File: RedisConfig.java package com.stackscout.config; public class RedisConfig { diff --git a/src/main/java/com/stackscout/config/SecurityConfig.java b/src/main/java/com/stackscout/config/SecurityConfig.java index 1cdc5c1..11f2d0b 100644 --- a/src/main/java/com/stackscout/config/SecurityConfig.java +++ b/src/main/java/com/stackscout/config/SecurityConfig.java @@ -1,3 +1,4 @@ +// File: SecurityConfig.java package com.stackscout.config; import org.springframework.context.annotation.Bean; diff --git a/src/main/java/com/stackscout/controller/CollectorController.java b/src/main/java/com/stackscout/controller/CollectorController.java index 5bb3250..1c85866 100644 --- a/src/main/java/com/stackscout/controller/CollectorController.java +++ b/src/main/java/com/stackscout/controller/CollectorController.java @@ -1,3 +1,4 @@ +// File: CollectorController.java package com.stackscout.controller; import com.stackscout.dto.ErrorResponse; diff --git a/src/main/java/com/stackscout/controller/HealthCheckController.java b/src/main/java/com/stackscout/controller/HealthCheckController.java index 67aeea2..a4be406 100644 --- a/src/main/java/com/stackscout/controller/HealthCheckController.java +++ b/src/main/java/com/stackscout/controller/HealthCheckController.java @@ -1,3 +1,4 @@ +// File: HealthCheckController.java package com.stackscout.controller; import org.springframework.http.ResponseEntity; diff --git a/src/main/java/com/stackscout/controller/LibraryController.java b/src/main/java/com/stackscout/controller/LibraryController.java index 7a12adc..9476a35 100644 --- a/src/main/java/com/stackscout/controller/LibraryController.java +++ b/src/main/java/com/stackscout/controller/LibraryController.java @@ -1,3 +1,4 @@ +// File: LibraryController.java package com.stackscout.controller; import com.stackscout.dto.CreateLibraryRequest; diff --git a/src/main/java/com/stackscout/controller/LicenseController.java b/src/main/java/com/stackscout/controller/LicenseController.java index a5393d1..0598d8c 100644 --- a/src/main/java/com/stackscout/controller/LicenseController.java +++ b/src/main/java/com/stackscout/controller/LicenseController.java @@ -1,3 +1,4 @@ +// File: LicenseController.java package com.stackscout.controller; import com.stackscout.dto.CreateLicenseRequest; diff --git a/src/main/java/com/stackscout/controller/ScanJobController.java b/src/main/java/com/stackscout/controller/ScanJobController.java index 9e4ab7d..6882f6a 100644 --- a/src/main/java/com/stackscout/controller/ScanJobController.java +++ b/src/main/java/com/stackscout/controller/ScanJobController.java @@ -1,3 +1,4 @@ +// File: ScanJobController.java package com.stackscout.controller; import com.stackscout.dto.CreateScanJobRequest; diff --git a/src/main/java/com/stackscout/dto/CreateLibraryRequest.java b/src/main/java/com/stackscout/dto/CreateLibraryRequest.java index f8f7d7c..03bfe10 100644 --- a/src/main/java/com/stackscout/dto/CreateLibraryRequest.java +++ b/src/main/java/com/stackscout/dto/CreateLibraryRequest.java @@ -1,3 +1,4 @@ +// File: CreateLibraryRequest.java package com.stackscout.dto; import jakarta.validation.constraints.Min; diff --git a/src/main/java/com/stackscout/dto/CreateLicenseRequest.java b/src/main/java/com/stackscout/dto/CreateLicenseRequest.java index 5964e13..614fde9 100644 --- a/src/main/java/com/stackscout/dto/CreateLicenseRequest.java +++ b/src/main/java/com/stackscout/dto/CreateLicenseRequest.java @@ -1,3 +1,4 @@ +// File: CreateLicenseRequest.java package com.stackscout.dto; import com.stackscout.model.License; diff --git a/src/main/java/com/stackscout/dto/CreateScanJobRequest.java b/src/main/java/com/stackscout/dto/CreateScanJobRequest.java index 884e2b4..f08be5d 100644 --- a/src/main/java/com/stackscout/dto/CreateScanJobRequest.java +++ b/src/main/java/com/stackscout/dto/CreateScanJobRequest.java @@ -1,3 +1,4 @@ +// File: CreateScanJobRequest.java package com.stackscout.dto; import jakarta.validation.constraints.NotBlank; diff --git a/src/main/java/com/stackscout/dto/ErrorResponse.java b/src/main/java/com/stackscout/dto/ErrorResponse.java index 324b8df..2e87e3f 100644 --- a/src/main/java/com/stackscout/dto/ErrorResponse.java +++ b/src/main/java/com/stackscout/dto/ErrorResponse.java @@ -1,3 +1,4 @@ +// File: ErrorResponse.java package com.stackscout.dto; import lombok.AllArgsConstructor; diff --git a/src/main/java/com/stackscout/dto/LibraryDto.java b/src/main/java/com/stackscout/dto/LibraryDto.java index ee27951..308ce5f 100644 --- a/src/main/java/com/stackscout/dto/LibraryDto.java +++ b/src/main/java/com/stackscout/dto/LibraryDto.java @@ -1,3 +1,4 @@ +// File: LibraryDto.java package com.stackscout.dto; import lombok.AllArgsConstructor; diff --git a/src/main/java/com/stackscout/dto/LibrarySearchRequest.java b/src/main/java/com/stackscout/dto/LibrarySearchRequest.java index 68279c9..7c7e336 100644 --- a/src/main/java/com/stackscout/dto/LibrarySearchRequest.java +++ b/src/main/java/com/stackscout/dto/LibrarySearchRequest.java @@ -1,3 +1,4 @@ +// File: LibrarySearchRequest.java package com.stackscout.dto; public class LibrarySearchRequest { diff --git a/src/main/java/com/stackscout/dto/LicenseDto.java b/src/main/java/com/stackscout/dto/LicenseDto.java index 12252c5..d0b64ad 100644 --- a/src/main/java/com/stackscout/dto/LicenseDto.java +++ b/src/main/java/com/stackscout/dto/LicenseDto.java @@ -1,3 +1,4 @@ +// File: LicenseDto.java package com.stackscout.dto; import com.stackscout.model.License; diff --git a/src/main/java/com/stackscout/dto/ScanJobDto.java b/src/main/java/com/stackscout/dto/ScanJobDto.java index d918f89..407055c 100644 --- a/src/main/java/com/stackscout/dto/ScanJobDto.java +++ b/src/main/java/com/stackscout/dto/ScanJobDto.java @@ -1,3 +1,4 @@ +// File: ScanJobDto.java package com.stackscout.dto; import com.stackscout.model.ScanJob; diff --git a/src/main/java/com/stackscout/dto/ScanRequest.java b/src/main/java/com/stackscout/dto/ScanRequest.java index 138127c..c2b4ea0 100644 --- a/src/main/java/com/stackscout/dto/ScanRequest.java +++ b/src/main/java/com/stackscout/dto/ScanRequest.java @@ -1,3 +1,4 @@ +// File: ScanRequest.java package com.stackscout.dto; import lombok.AllArgsConstructor; diff --git a/src/main/java/com/stackscout/dto/UpdateLibraryRequest.java b/src/main/java/com/stackscout/dto/UpdateLibraryRequest.java index c48b80a..12c3c31 100644 --- a/src/main/java/com/stackscout/dto/UpdateLibraryRequest.java +++ b/src/main/java/com/stackscout/dto/UpdateLibraryRequest.java @@ -1,3 +1,4 @@ +// File: UpdateLibraryRequest.java package com.stackscout.dto; import jakarta.validation.constraints.Min; diff --git a/src/main/java/com/stackscout/dto/dockerhub/DockerHubDTOs.java b/src/main/java/com/stackscout/dto/dockerhub/DockerHubDTOs.java new file mode 100644 index 0000000..ea90189 --- /dev/null +++ b/src/main/java/com/stackscout/dto/dockerhub/DockerHubDTOs.java @@ -0,0 +1,19 @@ +// File: DockerHubDTOs.java +package com.stackscout.dto.dockerhub; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.LocalDateTime; + +public class DockerHubDTOs { + + public record DockerHubRepository( + String user, + String name, + String namespace, + String description, + @JsonProperty("star_count") int starCount, + @JsonProperty("pull_count") long pullCount, + @JsonProperty("last_updated") LocalDateTime lastUpdated, + @JsonProperty("hub_user") String hubUser) { + } +} diff --git a/src/main/java/com/stackscout/dto/pypi/PyPiDTOs.java b/src/main/java/com/stackscout/dto/pypi/PyPiDTOs.java new file mode 100644 index 0000000..34a6d3f --- /dev/null +++ b/src/main/java/com/stackscout/dto/pypi/PyPiDTOs.java @@ -0,0 +1,27 @@ +// File: PyPiDTOs.java +package com.stackscout.dto.pypi; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; + +public class PyPiDTOs { + + public record PyPiResponse( + PyPiInfo info, + Map> releases) { + } + + public record PyPiInfo( + String name, + String version, + String summary, + String license, + @JsonProperty("home_page") String homePage, + @JsonProperty("project_urls") Map projectUrls) { + } + + public record PyPiRelease( + @JsonProperty("upload_time") String uploadTime) { + } +} diff --git a/src/main/java/com/stackscout/exception/BadRequestException.java b/src/main/java/com/stackscout/exception/BadRequestException.java index 8b8c9e9..62ea3eb 100644 --- a/src/main/java/com/stackscout/exception/BadRequestException.java +++ b/src/main/java/com/stackscout/exception/BadRequestException.java @@ -1,3 +1,4 @@ +// File: BadRequestException.java package com.stackscout.exception; public class BadRequestException { diff --git a/src/main/java/com/stackscout/exception/GlobalExceptionHandler.java b/src/main/java/com/stackscout/exception/GlobalExceptionHandler.java index 14b6ed0..bb171e1 100644 --- a/src/main/java/com/stackscout/exception/GlobalExceptionHandler.java +++ b/src/main/java/com/stackscout/exception/GlobalExceptionHandler.java @@ -1,3 +1,4 @@ +// File: GlobalExceptionHandler.java package com.stackscout.exception; public class GlobalExceptionHandler { diff --git a/src/main/java/com/stackscout/exception/ResourceNotFoundException.java b/src/main/java/com/stackscout/exception/ResourceNotFoundException.java index e8b543e..122d8da 100644 --- a/src/main/java/com/stackscout/exception/ResourceNotFoundException.java +++ b/src/main/java/com/stackscout/exception/ResourceNotFoundException.java @@ -1,3 +1,4 @@ +// File: ResourceNotFoundException.java package com.stackscout.exception; /** diff --git a/src/main/java/com/stackscout/exception/UnauthorizedException.java b/src/main/java/com/stackscout/exception/UnauthorizedException.java index 6b7bec5..8c64653 100644 --- a/src/main/java/com/stackscout/exception/UnauthorizedException.java +++ b/src/main/java/com/stackscout/exception/UnauthorizedException.java @@ -1,3 +1,4 @@ +// File: UnauthorizedException.java package com.stackscout.exception; public class UnauthorizedException { diff --git a/src/main/java/com/stackscout/mapper/LibraryMapper.java b/src/main/java/com/stackscout/mapper/LibraryMapper.java index 8963cf4..97eaa37 100644 --- a/src/main/java/com/stackscout/mapper/LibraryMapper.java +++ b/src/main/java/com/stackscout/mapper/LibraryMapper.java @@ -1,3 +1,4 @@ +// File: LibraryMapper.java package com.stackscout.mapper; import com.stackscout.dto.CreateLibraryRequest; diff --git a/src/main/java/com/stackscout/model/Library.java b/src/main/java/com/stackscout/model/Library.java index e61599f..6c608e1 100644 --- a/src/main/java/com/stackscout/model/Library.java +++ b/src/main/java/com/stackscout/model/Library.java @@ -1,3 +1,4 @@ +// File: Library.java package com.stackscout.model; import jakarta.persistence.*; diff --git a/src/main/java/com/stackscout/model/License.java b/src/main/java/com/stackscout/model/License.java index 7e44129..bd6bc48 100644 --- a/src/main/java/com/stackscout/model/License.java +++ b/src/main/java/com/stackscout/model/License.java @@ -1,3 +1,4 @@ +// File: License.java package com.stackscout.model; import jakarta.persistence.*; diff --git a/src/main/java/com/stackscout/model/Role.java b/src/main/java/com/stackscout/model/Role.java index 357867f..8fbd99b 100644 --- a/src/main/java/com/stackscout/model/Role.java +++ b/src/main/java/com/stackscout/model/Role.java @@ -1,3 +1,4 @@ +// File: Role.java package com.stackscout.model; public class Role { diff --git a/src/main/java/com/stackscout/model/ScanJob.java b/src/main/java/com/stackscout/model/ScanJob.java index ff85917..05322fb 100644 --- a/src/main/java/com/stackscout/model/ScanJob.java +++ b/src/main/java/com/stackscout/model/ScanJob.java @@ -1,3 +1,4 @@ +// File: ScanJob.java package com.stackscout.model; import jakarta.persistence.*; diff --git a/src/main/java/com/stackscout/repository/LibraryRepository.java b/src/main/java/com/stackscout/repository/LibraryRepository.java index 43a767f..d58810d 100644 --- a/src/main/java/com/stackscout/repository/LibraryRepository.java +++ b/src/main/java/com/stackscout/repository/LibraryRepository.java @@ -1,3 +1,4 @@ +// File: LibraryRepository.java package com.stackscout.repository; import com.stackscout.model.Library; diff --git a/src/main/java/com/stackscout/repository/LicenseRepository.java b/src/main/java/com/stackscout/repository/LicenseRepository.java index d0cdf02..661ea45 100644 --- a/src/main/java/com/stackscout/repository/LicenseRepository.java +++ b/src/main/java/com/stackscout/repository/LicenseRepository.java @@ -1,3 +1,4 @@ +// File: LicenseRepository.java package com.stackscout.repository; import com.stackscout.model.License; diff --git a/src/main/java/com/stackscout/repository/ScanJobRepository.java b/src/main/java/com/stackscout/repository/ScanJobRepository.java index d8a5c76..9b0113b 100644 --- a/src/main/java/com/stackscout/repository/ScanJobRepository.java +++ b/src/main/java/com/stackscout/repository/ScanJobRepository.java @@ -1,3 +1,4 @@ +// File: ScanJobRepository.java package com.stackscout.repository; import com.stackscout.model.ScanJob; diff --git a/src/main/java/com/stackscout/scheduler/CollectorScheduler.java b/src/main/java/com/stackscout/scheduler/CollectorScheduler.java index 5a1c3d0..60f72e4 100644 --- a/src/main/java/com/stackscout/scheduler/CollectorScheduler.java +++ b/src/main/java/com/stackscout/scheduler/CollectorScheduler.java @@ -1,3 +1,4 @@ +// File: CollectorScheduler.java package com.stackscout.scheduler; public class CollectorScheduler { diff --git a/src/main/java/com/stackscout/service/CollectorService.java b/src/main/java/com/stackscout/service/CollectorService.java index 5e14c67..dc75fde 100644 --- a/src/main/java/com/stackscout/service/CollectorService.java +++ b/src/main/java/com/stackscout/service/CollectorService.java @@ -1,5 +1,28 @@ +// File: CollectorService.java package com.stackscout.service; -public class CollectorService { - +import com.stackscout.model.Library; +import java.util.List; + +/** + * Service for collecting and processing library metadata from various sources. + */ +public interface CollectorService { + + /** + * Collect metadata for a single package. + * + * @param source The source (e.g., "pypi", "dockerhub") + * @param name The package name + * @return The processed and saved Library entity + */ + Library collect(String source, String name); + + /** + * Collect metadata for multiple packages. + * + * @param source The source + * @param names List of package names + */ + void collectBulk(String source, List names); } diff --git a/src/main/java/com/stackscout/service/DockerHubService.java b/src/main/java/com/stackscout/service/DockerHubService.java index a4d129b..49ba1c1 100644 --- a/src/main/java/com/stackscout/service/DockerHubService.java +++ b/src/main/java/com/stackscout/service/DockerHubService.java @@ -1,5 +1,19 @@ +// File: DockerHubService.java package com.stackscout.service; -public class DockerHubService { - +import com.stackscout.model.Library; +import java.util.Optional; + +/** + * Service for interacting with the Docker Hub V2 API to fetch image metadata. + */ +public interface DockerHubService { + /** + * Fetches metadata for a given Docker image. + * + * @param imageName The name of the image (e.g., "nginx" or + * "mysql/mysql-server"). + * @return Optional containing Library entity with metadata. + */ + Optional getImageInfo(String imageName); } diff --git a/src/main/java/com/stackscout/service/HealthScoreService.java b/src/main/java/com/stackscout/service/HealthScoreService.java index 007428b..ad8a147 100644 --- a/src/main/java/com/stackscout/service/HealthScoreService.java +++ b/src/main/java/com/stackscout/service/HealthScoreService.java @@ -1,5 +1,18 @@ +// File: HealthScoreService.java package com.stackscout.service; -public class HealthScoreService { - +import com.stackscout.model.Library; + +/** + * Service for calculating the health score of a library based on various + * metrics. + */ +public interface HealthScoreService { + /** + * Calculates a health score (0-100) for the given library. + * + * @param library The library entity to evaluate. + * @return The health score. + */ + int calculateScore(Library library); } diff --git a/src/main/java/com/stackscout/service/LibraryService.java b/src/main/java/com/stackscout/service/LibraryService.java index c4a4c44..f2e4403 100644 --- a/src/main/java/com/stackscout/service/LibraryService.java +++ b/src/main/java/com/stackscout/service/LibraryService.java @@ -1,3 +1,4 @@ +// File: LibraryService.java package com.stackscout.service; import com.stackscout.dto.CreateLibraryRequest; diff --git a/src/main/java/com/stackscout/service/LicenseService.java b/src/main/java/com/stackscout/service/LicenseService.java index 9b38908..14a2e34 100644 --- a/src/main/java/com/stackscout/service/LicenseService.java +++ b/src/main/java/com/stackscout/service/LicenseService.java @@ -1,3 +1,4 @@ +// File: LicenseService.java package com.stackscout.service; import com.stackscout.dto.CreateLicenseRequest; @@ -12,44 +13,49 @@ * Сервис для работы с лицензиями */ public interface LicenseService { - + /** * Получить все лицензии с пагинацией */ Page getAllLicenses(Pageable pageable); - + /** * Получить лицензию по ID */ LicenseDto getLicenseById(Long id); - + /** * Создать новую лицензию */ LicenseDto createLicense(CreateLicenseRequest request); - + /** * Обновить существующую лицензию */ LicenseDto updateLicense(Long id, CreateLicenseRequest request); - + /** * Удалить лицензию */ void deleteLicense(Long id); - + /** * Найти лицензию по имени */ LicenseDto findByName(String name); - + /** * Получить лицензии по типу */ List getLicensesByType(License.LicenseType type); - + /** * Получить OSI-одобренные лицензии */ List getOsiApprovedLicenses(); + + /** + * Нормализовать название лицензии + */ + String normalize(String licenseName); } diff --git a/src/main/java/com/stackscout/service/PyPiService.java b/src/main/java/com/stackscout/service/PyPiService.java index c6498ac..9acbff6 100644 --- a/src/main/java/com/stackscout/service/PyPiService.java +++ b/src/main/java/com/stackscout/service/PyPiService.java @@ -1,5 +1,18 @@ +// File: PyPiService.java package com.stackscout.service; -public class PyPiService { - +import com.stackscout.model.Library; +import java.util.Optional; + +/** + * Service for interacting with the PyPI JSON API to fetch package metadata. + */ +public interface PyPiService { + /** + * Fetches metadata for a given Python package. + * + * @param packageName The name of the package. + * @return Optional containing Library entity with metadata. + */ + Optional getPackageInfo(String packageName); } diff --git a/src/main/java/com/stackscout/service/ScanJobService.java b/src/main/java/com/stackscout/service/ScanJobService.java index a938fcc..798cf19 100644 --- a/src/main/java/com/stackscout/service/ScanJobService.java +++ b/src/main/java/com/stackscout/service/ScanJobService.java @@ -1,3 +1,4 @@ +// File: ScanJobService.java package com.stackscout.service; import com.stackscout.dto.CreateScanJobRequest; diff --git a/src/main/java/com/stackscout/service/impl/CollectorServiceImpl.java b/src/main/java/com/stackscout/service/impl/CollectorServiceImpl.java index d33e0c0..82170c8 100644 --- a/src/main/java/com/stackscout/service/impl/CollectorServiceImpl.java +++ b/src/main/java/com/stackscout/service/impl/CollectorServiceImpl.java @@ -1,5 +1,97 @@ +// File: CollectorServiceImpl.java package com.stackscout.service.impl; -public class CollectorServiceImpl { - +import com.stackscout.model.Library; +import com.stackscout.repository.LibraryRepository; +import com.stackscout.service.*; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +/** + * Orchestrator service that coordinates collection from various sources, + * normalizes data, calculates scores, and persists results. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class CollectorServiceImpl implements CollectorService { + + private final PyPiService pyPiService; + private final DockerHubService dockerHubService; + private final LicenseService licenseService; + private final HealthScoreService healthScoreService; + private final LibraryRepository libraryRepository; + + @Override + @Transactional + public Library collect(String source, String name) { + log.info("Collecting metadata for {} from {}", name, source); + + Optional optionalLibrary = switch (source.toLowerCase()) { + case "pypi" -> pyPiService.getPackageInfo(name); + case "docker", "dockerhub" -> dockerHubService.getImageInfo(name); + default -> { + log.warn("Unsupported source: {}", source); + yield Optional.empty(); + } + }; + + if (optionalLibrary.isEmpty()) { + log.warn("Could not find metadata for {} in {}", name, source); + return null; + } + + Library library = optionalLibrary.get(); + + // 1. Normalize License + if (library.getLicense() != null) { + String normalizedLicense = licenseService.normalize(library.getLicense()); + library.setLicense(normalizedLicense); + } + + // 2. Calculate Health Score + int score = healthScoreService.calculateScore(library); + library.setHealthScore(score); + + // 3. Persist (Update if exists, or Save new) + return saveOrUpdate(library); + } + + @Override + @Transactional + public void collectBulk(String source, List names) { + log.info("Starting bulk collection for {} packages from {}", names.size(), source); + for (String name : names) { + try { + collect(source, name); + } catch (Exception e) { + log.error("Failed to collect {} from {}: {}", name, source, e.getMessage()); + } + } + } + + private Library saveOrUpdate(Library library) { + Optional existing = libraryRepository.findByName(library.getName()); + + if (existing.isPresent()) { + Library entity = existing.get(); + entity.setVersion(library.getVersion()); + entity.setSource(library.getSource()); + entity.setLicense(library.getLicense()); + entity.setHealthScore(library.getHealthScore()); + entity.setLastRelease(library.getLastRelease()); + entity.setRepository(library.getRepository()); + entity.setDescription(library.getDescription()); + entity.setUpdatedAt(LocalDateTime.now()); + return libraryRepository.save(entity); + } else { + return libraryRepository.save(library); + } + } } diff --git a/src/main/java/com/stackscout/service/impl/DockerHubServiceImpl.java b/src/main/java/com/stackscout/service/impl/DockerHubServiceImpl.java index 7578d84..8590958 100644 --- a/src/main/java/com/stackscout/service/impl/DockerHubServiceImpl.java +++ b/src/main/java/com/stackscout/service/impl/DockerHubServiceImpl.java @@ -1,5 +1,74 @@ +// File: DockerHubServiceImpl.java package com.stackscout.service.impl; -public class DockerHubServiceImpl { - +import com.stackscout.dto.dockerhub.DockerHubDTOs; +import com.stackscout.model.Library; +import com.stackscout.service.DockerHubService; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClient; + +import java.util.Optional; + +@Service +@RequiredArgsConstructor +public class DockerHubServiceImpl implements DockerHubService { + + private final RestClient restClient; + private static final Logger logger = LoggerFactory.getLogger(DockerHubServiceImpl.class); + + public DockerHubServiceImpl() { + this.restClient = RestClient.builder() + .baseUrl("https://hub.docker.com/v2/repositories") + .build(); + } + + // Constructor for testing injectability + public DockerHubServiceImpl(RestClient.Builder restClientBuilder) { + this.restClient = restClientBuilder + .baseUrl("https://hub.docker.com/v2/repositories") + .build(); + } + + @Override + public Optional getImageInfo(String imageName) { + String namespace = "library"; + String repository = imageName; + + if (imageName.contains("/")) { + String[] parts = imageName.split("/", 2); + namespace = parts[0]; + repository = parts[1]; + } + + try { + return Optional.ofNullable(restClient.get() + .uri("/{namespace}/{repository}", namespace, repository) + .retrieve() + .body(DockerHubDTOs.DockerHubRepository.class)) + .map(this::mapToLibrary); + } catch (Exception e) { + logger.warn("Failed to fetch Docker Hub image info for: {}/{}", namespace, repository, e); + return Optional.empty(); + } + } + + private Library mapToLibrary(DockerHubDTOs.DockerHubRepository repo) { + Library lib = new Library(); + lib.setName(repo.namespace().equals("library") ? repo.name() : repo.namespace() + "/" + repo.name()); + lib.setVersion("latest"); // Docker Hub API doesn't give a single version, defaulting to latest logic + lib.setSource("docker"); + lib.setLicense(""); // Docker Hub API doesn't consistently provide license in this endpoint + lib.setHealthScore(0); + lib.setDescription(repo.description()); + lib.setRepository("https://hub.docker.com/r/" + repo.namespace() + "/" + repo.name()); + + if (repo.lastUpdated() != null) { + lib.setLastRelease(repo.lastUpdated().toString()); + } + + return lib; + } } diff --git a/src/main/java/com/stackscout/service/impl/HealthScoreServiceImpl.java b/src/main/java/com/stackscout/service/impl/HealthScoreServiceImpl.java index e69de29..d5077c8 100644 --- a/src/main/java/com/stackscout/service/impl/HealthScoreServiceImpl.java +++ b/src/main/java/com/stackscout/service/impl/HealthScoreServiceImpl.java @@ -0,0 +1,74 @@ +// File: HealthScoreServiceImpl.java +package com.stackscout.service.impl; + +import com.stackscout.model.Library; +import com.stackscout.service.HealthScoreService; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; + +@Service +public class HealthScoreServiceImpl implements HealthScoreService { + + @Override + public int calculateScore(Library library) { + if (library == null) { + return 0; + } + + int score = 0; + + // 1. Recency (0-40 points) + score += calculateRecencyScore(library.getLastRelease()); + + // 2. Repository (0-20 points) + if (hasValue(library.getRepository())) { + score += 20; + } + + // 3. Documentation/Description (0-10 points) + if (hasValue(library.getDescription())) { + score += 10; + } + + // 4. License (0-10 points) - Basic check + if (hasValue(library.getLicense())) { + score += 10; + } + + // Future: Community activity, etc. + + return Math.min(100, Math.max(0, score)); + } + + private int calculateRecencyScore(String lastReleaseDate) { + if (lastReleaseDate == null || lastReleaseDate.isBlank()) { + return 0; + } + + try { + // Ry to parse flexible format, usually ISO from APIs + LocalDateTime releaseDate = LocalDateTime.parse(lastReleaseDate, DateTimeFormatter.ISO_DATE_TIME); + long daysDiff = ChronoUnit.DAYS.between(releaseDate, LocalDateTime.now()); + + if (daysDiff < 180) { // < 6 months + return 40; + } else if (daysDiff < 365) { // < 1 year + return 30; + } else if (daysDiff < 730) { // < 2 years + return 20; + } else { + return 5; + } + } catch (Exception e) { + // Parsing failed, return 0 or try other format + return 0; + } + } + + private boolean hasValue(String value) { + return value != null && !value.isBlank(); + } +} diff --git a/src/main/java/com/stackscout/service/impl/LibraryServiceImpl.java b/src/main/java/com/stackscout/service/impl/LibraryServiceImpl.java index d786b6f..ff1a0a9 100644 --- a/src/main/java/com/stackscout/service/impl/LibraryServiceImpl.java +++ b/src/main/java/com/stackscout/service/impl/LibraryServiceImpl.java @@ -1,3 +1,4 @@ +// File: LibraryServiceImpl.java package com.stackscout.service.impl; import com.stackscout.dto.CreateLibraryRequest; diff --git a/src/main/java/com/stackscout/service/impl/LicenseServiceImpl.java b/src/main/java/com/stackscout/service/impl/LicenseServiceImpl.java index 6b209c7..55d5fd5 100644 --- a/src/main/java/com/stackscout/service/impl/LicenseServiceImpl.java +++ b/src/main/java/com/stackscout/service/impl/LicenseServiceImpl.java @@ -1,3 +1,4 @@ +// File: LicenseServiceImpl.java package com.stackscout.service.impl; import com.stackscout.dto.CreateLicenseRequest; @@ -24,9 +25,9 @@ @RequiredArgsConstructor @Transactional(readOnly = true) public class LicenseServiceImpl implements LicenseService { - + private final LicenseRepository licenseRepository; - + @Override public Page getAllLicenses(Pageable pageable) { log.debug("Получение всех лицензий с пагинацией: {}", pageable); @@ -36,7 +37,7 @@ public Page getAllLicenses(Pageable pageable) { return licenseRepository.findAll(pageable) .map(this::toDto); } - + @Override public LicenseDto getLicenseById(Long id) { log.debug("Поиск лицензии с ID: {}", id); @@ -47,22 +48,22 @@ public LicenseDto getLicenseById(Long id) { .orElseThrow(() -> new ResourceNotFoundException("Лицензия не найдена с ID: " + id)); return toDto(license); } - + @Override @Transactional public LicenseDto createLicense(CreateLicenseRequest request) { log.info("Создание новой лицензии: {}", request.getName()); - + License license = toEntity(request); if (license == null) { throw new IllegalArgumentException("License не может быть null"); } License savedLicense = licenseRepository.save(license); - + log.info("Лицензия успешно создана с ID: {}", savedLicense.getId()); return toDto(savedLicense); } - + @Override @Transactional public LicenseDto updateLicense(Long id, CreateLicenseRequest request) { @@ -70,18 +71,18 @@ public LicenseDto updateLicense(Long id, CreateLicenseRequest request) { if (id == null) { throw new IllegalArgumentException("ID не может быть null"); } - + License license = licenseRepository.findById(id) .orElseThrow(() -> new ResourceNotFoundException("Лицензия не найдена с ID: " + id)); - + updateEntityFromDto(license, request); @SuppressWarnings("null") License updatedLicense = licenseRepository.save(license); - + log.info("Лицензия успешно обновлена: {}", id); return toDto(updatedLicense); } - + @Override @Transactional public void deleteLicense(Long id) { @@ -89,15 +90,15 @@ public void deleteLicense(Long id) { if (id == null) { throw new IllegalArgumentException("ID не может быть null"); } - + if (!licenseRepository.existsById(id)) { throw new ResourceNotFoundException("Лицензия не найдена с ID: " + id); } - + licenseRepository.deleteById(id); log.info("Лицензия успешно удалена: {}", id); } - + @Override public LicenseDto findByName(String name) { log.debug("Поиск лицензии по имени: {}", name); @@ -105,7 +106,7 @@ public LicenseDto findByName(String name) { .orElseThrow(() -> new ResourceNotFoundException("Лицензия не найдена с именем: " + name)); return toDto(license); } - + @Override public List getLicensesByType(License.LicenseType type) { log.debug("Получение лицензий по типу: {}", type); @@ -114,7 +115,7 @@ public List getLicensesByType(License.LicenseType type) { .map(this::toDto) .collect(Collectors.toList()); } - + @Override public List getOsiApprovedLicenses() { log.debug("Получение OSI-одобренных лицензий"); @@ -123,23 +124,22 @@ public List getOsiApprovedLicenses() { .map(this::toDto) .collect(Collectors.toList()); } - + // Вспомогательные методы для маппинга - + private LicenseDto toDto(License license) { return new LicenseDto( - license.getId(), - license.getName(), - license.getLicenseType(), - license.getDescription(), - license.getIsOsiApproved(), - license.getCommercialUseAllowed(), - license.getUrl(), - license.getCreatedAt(), - license.getUpdatedAt() - ); + license.getId(), + license.getName(), + license.getLicenseType(), + license.getDescription(), + license.getIsOsiApproved(), + license.getCommercialUseAllowed(), + license.getUrl(), + license.getCreatedAt(), + license.getUpdatedAt()); } - + private License toEntity(CreateLicenseRequest request) { License license = new License(); license.setName(request.getName()); @@ -150,7 +150,7 @@ private License toEntity(CreateLicenseRequest request) { license.setUrl(request.getUrl()); return license; } - + private void updateEntityFromDto(License license, CreateLicenseRequest request) { if (request.getName() != null) { license.setName(request.getName()); @@ -171,4 +171,29 @@ private void updateEntityFromDto(License license, CreateLicenseRequest request) license.setUrl(request.getUrl()); } } + + @Override + public String normalize(String licenseName) { + if (licenseName == null || licenseName.isBlank()) { + return "Unknown"; + } + + String normalized = licenseName.trim(); + + // Common variations map + if (normalized.equalsIgnoreCase("MIT License") || normalized.equalsIgnoreCase("The MIT License")) { + return "MIT"; + } + if (normalized.equalsIgnoreCase("Apache License 2.0") || normalized.equalsIgnoreCase("Apache 2.0")) { + return "Apache-2.0"; + } + if (normalized.equalsIgnoreCase("GNU General Public License v3.0") || normalized.equalsIgnoreCase("GPLv3")) { + return "GPL-3.0"; + } + if (normalized.equalsIgnoreCase("BSD 3-Clause \"New\" or \"Revised\" License")) { + return "BSD-3-Clause"; + } + + return normalized; + } } diff --git a/src/main/java/com/stackscout/service/impl/PyPiServiceImpl.java b/src/main/java/com/stackscout/service/impl/PyPiServiceImpl.java index 44b8fc7..f408c0f 100644 --- a/src/main/java/com/stackscout/service/impl/PyPiServiceImpl.java +++ b/src/main/java/com/stackscout/service/impl/PyPiServiceImpl.java @@ -1,5 +1,87 @@ +// File: PyPiServiceImpl.java package com.stackscout.service.impl; -public class PyPiServiceImpl { - +import com.stackscout.dto.pypi.PyPiDTOs; +import com.stackscout.model.Library; +import com.stackscout.service.PyPiService; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Optional; + +@Service +@RequiredArgsConstructor +public class PyPiServiceImpl implements PyPiService { + + private final RestClient restClient; + private static final Logger logger = LoggerFactory.getLogger(PyPiServiceImpl.class); + + public PyPiServiceImpl() { + this.restClient = RestClient.builder() + .baseUrl("https://pypi.org/pypi") + .build(); + } + + // Constructor for testing injectability + public PyPiServiceImpl(RestClient.Builder restClientBuilder) { + this.restClient = restClientBuilder + .baseUrl("https://pypi.org/pypi") + .build(); + } + + @Override + public Optional getPackageInfo(String packageName) { + try { + return Optional.ofNullable(restClient.get() + .uri("/{package}/json", packageName) + .retrieve() + .body(PyPiDTOs.PyPiResponse.class)) + .map(this::mapToLibrary); + } catch (Exception e) { + logger.warn("Failed to fetch PyPI package info for: {}", packageName, e); + return Optional.empty(); + } + } + + private Library mapToLibrary(PyPiDTOs.PyPiResponse response) { + PyPiDTOs.PyPiInfo info = response.info(); + + Library lib = new Library(); + lib.setName(info.name()); + lib.setVersion(info.version()); + lib.setSource("pypi"); + lib.setLicense(info.license()); + lib.setHealthScore(0); // Calculator will be implemented later + lib.setDescription(info.summary()); + + // Find repository URL + String repoUrl = info.homePage(); + if (info.projectUrls() != null) { + for (Map.Entry entry : info.projectUrls().entrySet()) { + String key = entry.getKey().toLowerCase(); + String value = entry.getValue(); + if (key.contains("source") || key.contains("repository") || key.contains("github") + || key.contains("gitlab")) { + repoUrl = value; + break; + } + } + } + lib.setRepository(repoUrl); + + // Find last release date + if (response.releases() != null && response.releases().containsKey(info.version())) { + var releases = response.releases().get(info.version()); + if (releases != null && !releases.isEmpty()) { + String uploadTime = releases.get(0).uploadTime(); + lib.setLastRelease(uploadTime); + } + } + + return lib; + } } diff --git a/src/main/java/com/stackscout/service/impl/ScanJobServiceImpl.java b/src/main/java/com/stackscout/service/impl/ScanJobServiceImpl.java index 82794ab..d9e35cd 100644 --- a/src/main/java/com/stackscout/service/impl/ScanJobServiceImpl.java +++ b/src/main/java/com/stackscout/service/impl/ScanJobServiceImpl.java @@ -1,3 +1,4 @@ +// File: ScanJobServiceImpl.java package com.stackscout.service.impl; import com.stackscout.dto.CreateScanJobRequest; diff --git a/src/main/java/com/stackscout/util/DateUtil.java b/src/main/java/com/stackscout/util/DateUtil.java index 80dad61..a393b1e 100644 --- a/src/main/java/com/stackscout/util/DateUtil.java +++ b/src/main/java/com/stackscout/util/DateUtil.java @@ -1,3 +1,4 @@ +// File: DateUtil.java package com.stackscout.util; public class DateUtil { diff --git a/src/main/java/com/stackscout/util/ValidationUtil.java b/src/main/java/com/stackscout/util/ValidationUtil.java index 6edcf2c..2676f1e 100644 --- a/src/main/java/com/stackscout/util/ValidationUtil.java +++ b/src/main/java/com/stackscout/util/ValidationUtil.java @@ -1,3 +1,4 @@ +// File: ValidationUtil.java package com.stackscout.util; public class ValidationUtil { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 2053ec6..2ee4e39 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -4,16 +4,13 @@ server: spring: application: name: StackScout - - profiles: - active: dev - + datasource: url: jdbc:postgresql://localhost:5432/stackscout username: postgres password: postgres driver-class-name: org.postgresql.Driver - + jpa: hibernate: ddl-auto: validate @@ -21,25 +18,25 @@ spring: properties: hibernate: dialect: org.hibernate.dialect.PostgreSQLDialect - '[format_sql]': true - + "[format_sql]": true + flyway: enabled: true baseline-on-migrate: true locations: classpath:db/migration - + data: redis: host: localhost port: 6379 - password: - + password: + rabbitmq: host: localhost port: 5672 username: guest password: guest - + cache: type: redis redis: @@ -48,4 +45,4 @@ spring: logging: level: root: INFO - '[com.stackscout]': DEBUG + "[com.stackscout]": DEBUG diff --git a/src/main/resources/db/migration/V1__init_schema.sql b/src/main/resources/db/migration/V1__init_schema.sql index e69de29..476f7b1 100644 --- a/src/main/resources/db/migration/V1__init_schema.sql +++ b/src/main/resources/db/migration/V1__init_schema.sql @@ -0,0 +1,17 @@ +CREATE TABLE libraries ( + id bigserial PRIMARY KEY, + name varchar(255) NOT NULL, + version varchar(50) NOT NULL, + source varchar(50) NOT NULL, + license_name varchar(100), + health_score integer, + last_release varchar(50), + repository varchar(500), + description text, + created_at timestamp NOT NULL, + updated_at timestamp +); + +CREATE INDEX idx_library_name ON libraries(name); +CREATE INDEX idx_library_source ON libraries(source); +CREATE INDEX idx_library_health_score ON libraries(health_score); diff --git a/src/main/resources/db/migration/V2__add_licenses.sql b/src/main/resources/db/migration/V2__add_licenses.sql index e69de29..c27c254 100644 --- a/src/main/resources/db/migration/V2__add_licenses.sql +++ b/src/main/resources/db/migration/V2__add_licenses.sql @@ -0,0 +1,14 @@ +CREATE TABLE licenses ( + id bigserial PRIMARY KEY, + name varchar(100) NOT NULL UNIQUE, + license_type varchar(50) NOT NULL, + description text, + is_osi_approved boolean, + commercial_use_allowed boolean, + url varchar(500), + created_at timestamp NOT NULL, + updated_at timestamp +); + +CREATE UNIQUE INDEX idx_license_name ON licenses(name); +CREATE INDEX idx_license_type ON licenses(license_type); diff --git a/src/main/resources/db/migration/V3__add_scan_jobs.sql b/src/main/resources/db/migration/V3__add_scan_jobs.sql index e69de29..866d9cf 100644 --- a/src/main/resources/db/migration/V3__add_scan_jobs.sql +++ b/src/main/resources/db/migration/V3__add_scan_jobs.sql @@ -0,0 +1,17 @@ +CREATE TABLE scan_jobs ( + id bigserial PRIMARY KEY, + source varchar(50) NOT NULL, + status varchar(50) NOT NULL, + packages_count integer, + processed_count integer, + failed_count integer, + started_at timestamp, + completed_at timestamp, + error_message text, + created_at timestamp NOT NULL, + updated_at timestamp +); + +CREATE INDEX idx_scan_job_status ON scan_jobs(status); +CREATE INDEX idx_scan_job_source ON scan_jobs(source); +CREATE INDEX idx_scan_job_created ON scan_jobs(created_at); diff --git a/src/test/java/com/stackscout/StackScoutApplicationTests.java b/src/test/java/com/stackscout/StackScoutApplicationTests.java index a773ef1..6cbd018 100644 --- a/src/test/java/com/stackscout/StackScoutApplicationTests.java +++ b/src/test/java/com/stackscout/StackScoutApplicationTests.java @@ -1,3 +1,4 @@ +// File: StackScoutApplicationTests.java package com.stackscout; import org.junit.jupiter.api.Test; diff --git a/src/test/java/com/stackscout/service/impl/DockerHubServiceImplTest.java b/src/test/java/com/stackscout/service/impl/DockerHubServiceImplTest.java new file mode 100644 index 0000000..aaa2167 --- /dev/null +++ b/src/test/java/com/stackscout/service/impl/DockerHubServiceImplTest.java @@ -0,0 +1,102 @@ +// File: DockerHubServiceImplTest.java +package com.stackscout.service.impl; + +import com.stackscout.model.Library; +import com.stackscout.service.DockerHubService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestClient; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withResourceNotFound; + +class DockerHubServiceImplTest { + + private DockerHubService dockerHubService; + private MockRestServiceServer mockServer; + + @BeforeEach + void setUp() { + RestClient.Builder builder = RestClient.builder(); + mockServer = MockRestServiceServer.bindTo(builder).build(); + dockerHubService = new DockerHubServiceImpl(builder); + } + + @Test + void getImageInfo_ShouldReturnLibrary_WhenImageExists_Official() { + // Arrange + String jsonResponse = """ + { + "user": "library", + "name": "nginx", + "namespace": "library", + "description": "Official build of Nginx.", + "star_count": 1000, + "pull_count": 5000000, + "last_updated": "2024-01-20T10:00:00.000Z", + "hub_user": "library" + } + """; + + mockServer.expect(requestTo("https://hub.docker.com/v2/repositories/library/nginx")) + .andRespond(withSuccess(jsonResponse, MediaType.APPLICATION_JSON)); + + // Act + Optional result = dockerHubService.getImageInfo("nginx"); + + // Assert + assertTrue(result.isPresent()); + Library lib = result.get(); + assertEquals("nginx", lib.getName()); + assertEquals("docker", lib.getSource()); + assertEquals("Official build of Nginx.", lib.getDescription()); + assertEquals("https://hub.docker.com/r/library/nginx", lib.getRepository()); + } + + @Test + void getImageInfo_ShouldReturnLibrary_WhenImageExists_Custom() { + // Arrange + String jsonResponse = """ + { + "user": "mysql", + "name": "mysql-server", + "namespace": "mysql", + "description": "MySQL Server", + "star_count": 500, + "pull_count": 100000, + "last_updated": "2024-01-15T12:00:00.000Z", + "hub_user": "mysql" + } + """; + + mockServer.expect(requestTo("https://hub.docker.com/v2/repositories/mysql/mysql-server")) + .andRespond(withSuccess(jsonResponse, MediaType.APPLICATION_JSON)); + + // Act + Optional result = dockerHubService.getImageInfo("mysql/mysql-server"); + + // Assert + assertTrue(result.isPresent()); + Library lib = result.get(); + assertEquals("mysql/mysql-server", lib.getName()); + } + + @Test + void getImageInfo_ShouldReturnEmpty_WhenImageNotFound() { + // Arrange + mockServer.expect(requestTo("https://hub.docker.com/v2/repositories/library/unknown")) + .andRespond(withResourceNotFound()); + + // Act + Optional result = dockerHubService.getImageInfo("unknown"); + + // Assert + assertTrue(result.isEmpty()); + } +} diff --git a/src/test/java/com/stackscout/service/impl/HealthScoreServiceImplTest.java b/src/test/java/com/stackscout/service/impl/HealthScoreServiceImplTest.java new file mode 100644 index 0000000..34b3866 --- /dev/null +++ b/src/test/java/com/stackscout/service/impl/HealthScoreServiceImplTest.java @@ -0,0 +1,51 @@ +// File: HealthScoreServiceImplTest.java +package com.stackscout.service.impl; + +import com.stackscout.model.Library; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class HealthScoreServiceImplTest { + + private HealthScoreServiceImpl healthScoreService; + + @BeforeEach + void setUp() { + healthScoreService = new HealthScoreServiceImpl(); + } + + @Test + void calculateScore_ShouldReturnMaxScore_ForPerfectLibrary() { + Library lib = new Library(); + lib.setLastRelease(LocalDateTime.now().minusDays(10).format(DateTimeFormatter.ISO_DATE_TIME)); + lib.setRepository("https://github.com/test/repo"); + lib.setDescription("A great library"); + lib.setLicense("MIT"); + + // 40 + 20 + 10 + 10 = 80 + assertEquals(80, healthScoreService.calculateScore(lib)); + } + + @Test + void calculateScore_ShouldReturnScore_ForOlderLibrary() { + Library lib = new Library(); + lib.setLastRelease(LocalDateTime.now().minusDays(400).format(DateTimeFormatter.ISO_DATE_TIME)); // > 1 year + lib.setRepository("https://github.com/test/repo"); + lib.setDescription("A great library"); + lib.setLicense("MIT"); + + // 20 (Recency) + 20 + 10 + 10 = 60 + assertEquals(60, healthScoreService.calculateScore(lib)); + } + + @Test + void calculateScore_ShouldReturnZero_ForEmptyLibrary() { + Library lib = new Library(); + assertEquals(0, healthScoreService.calculateScore(lib)); + } +} diff --git a/src/test/java/com/stackscout/service/impl/LicenseServiceImplTest.java b/src/test/java/com/stackscout/service/impl/LicenseServiceImplTest.java new file mode 100644 index 0000000..51583b1 --- /dev/null +++ b/src/test/java/com/stackscout/service/impl/LicenseServiceImplTest.java @@ -0,0 +1,42 @@ +// File: LicenseServiceImplTest.java +package com.stackscout.service.impl; + +import com.stackscout.repository.LicenseRepository; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@ExtendWith(MockitoExtension.class) +class LicenseServiceImplTest { + + @Mock + private LicenseRepository licenseRepository; + + @InjectMocks + private LicenseServiceImpl licenseService; + + @Test + void normalize_ShouldReturnUnknown_WhenNullOrBlank() { + assertEquals("Unknown", licenseService.normalize(null)); + assertEquals("Unknown", licenseService.normalize("")); + assertEquals("Unknown", licenseService.normalize(" ")); + } + + @Test + void normalize_ShouldNormalizeCommonLicenses() { + assertEquals("MIT", licenseService.normalize("MIT License")); + assertEquals("MIT", licenseService.normalize("The MIT License")); + assertEquals("Apache-2.0", licenseService.normalize("Apache License 2.0")); + assertEquals("Apache-2.0", licenseService.normalize("Apache 2.0")); + assertEquals("GPL-3.0", licenseService.normalize("GPLv3")); + } + + @Test + void normalize_ShouldReturnOriginal_WhenNoMatch() { + assertEquals("Custom License", licenseService.normalize("Custom License")); + } +} diff --git a/src/test/java/com/stackscout/service/impl/PyPiServiceImplTest.java b/src/test/java/com/stackscout/service/impl/PyPiServiceImplTest.java new file mode 100644 index 0000000..4a603f4 --- /dev/null +++ b/src/test/java/com/stackscout/service/impl/PyPiServiceImplTest.java @@ -0,0 +1,84 @@ +// File: PyPiServiceImplTest.java +package com.stackscout.service.impl; + +import com.stackscout.model.Library; +import com.stackscout.service.PyPiService; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestClient; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withResourceNotFound; + +class PyPiServiceImplTest { + + private PyPiService pyPiService; + private MockRestServiceServer mockServer; + + @BeforeEach + void setUp() { + RestClient.Builder builder = RestClient.builder(); + mockServer = MockRestServiceServer.bindTo(builder).build(); + pyPiService = new PyPiServiceImpl(builder); + } + + @Test + void getPackageInfo_ShouldReturnLibrary_WhenPackageExists() { + // Arrange + String jsonResponse = """ + { + "info": { + "name": "requests", + "version": "2.31.0", + "summary": "Python HTTP for Humans.", + "license": "Apache 2.0", + "home_page": "https://requests.readthedocs.io", + "project_urls": { + "Source": "https://github.com/psf/requests" + } + }, + "releases": { + "2.31.0": [ + { "upload_time": "2023-05-22T10:00:00" } + ] + } + } + """; + + mockServer.expect(requestTo("https://pypi.org/pypi/requests/json")) + .andRespond(withSuccess(jsonResponse, MediaType.APPLICATION_JSON)); + + // Act + Optional result = pyPiService.getPackageInfo("requests"); + + // Assert + assertTrue(result.isPresent()); + Library lib = result.get(); + assertEquals("requests", lib.getName()); + assertEquals("2.31.0", lib.getVersion()); + assertEquals("pypi", lib.getSource()); + assertEquals("Apache 2.0", lib.getLicense()); + assertEquals("https://github.com/psf/requests", lib.getRepository()); + assertEquals("2023-05-22T10:00:00", lib.getLastRelease()); + } + + @Test + void getPackageInfo_ShouldReturnEmpty_WhenPackageNotFound() { + // Arrange + mockServer.expect(requestTo("https://pypi.org/pypi/unknown-package/json")) + .andRespond(withResourceNotFound()); + + // Act + Optional result = pyPiService.getPackageInfo("unknown-package"); + + // Assert + assertTrue(result.isEmpty()); + } +} diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml deleted file mode 100644 index e69de29..0000000