diff --git a/bazarr/api/__init__.py b/bazarr/api/__init__.py index 3fa2eb81ed..0eca2e9396 100644 --- a/bazarr/api/__init__.py +++ b/bazarr/api/__init__.py @@ -10,6 +10,7 @@ from .movies import api_ns_list_movies from .providers import api_ns_list_providers from .series import api_ns_list_series +from .sports import api_ns_list_sports from .subtitles import api_ns_list_subtitles from .system import api_ns_list_system from .webhooks import api_ns_list_webhooks @@ -25,6 +26,7 @@ api_ns_list_movies, api_ns_list_providers, api_ns_list_series, + api_ns_list_sports, api_ns_list_subtitles, api_ns_list_system, api_ns_list_webhooks, diff --git a/bazarr/api/badges/badges.py b/bazarr/api/badges/badges.py index 2931c5e30f..3de60eeb27 100644 --- a/bazarr/api/badges/badges.py +++ b/bazarr/api/badges/badges.py @@ -6,11 +6,13 @@ from functools import reduce from flask_restx import Resource, Namespace, fields, marshal -from app.database import get_exclusion_clause, TableEpisodes, TableShows, TableMovies, database, select +from app.database import (get_exclusion_clause, TableEpisodes, TableShows, TableMovies, TableSportsEvents, + TableSportsLeagues, database, select) from app.config import settings from app.get_providers import get_throttled_providers from app.signalr_client import sonarr_signalr_client, radarr_signalr_client +from sportarr.sse_client import sportarr_sse_client from app.announcements import get_all_announcements from utilities.health import get_health_issues @@ -25,10 +27,12 @@ class Badges(Resource): get_model = api_ns_badges.model('BadgesGet', { 'episodes': fields.Integer(), 'movies': fields.Integer(), + 'sports': fields.Integer(), 'providers': fields.Integer(), 'status': fields.Integer(), 'sonarr_signalr': fields.String(), 'radarr_signalr': fields.String(), + 'sportarr_sse': fields.String(), 'announcements': fields.Integer(), }) @@ -62,6 +66,21 @@ def get(self): for movie in missing_movies: missing_movies_count += len(ast.literal_eval(movie.missing_subtitles)) + # Counted per playable file, because a sports row is one part. An event + # with two parts wants subtitles for both. + sports_conditions = [(TableSportsEvents.missing_subtitles.is_not(None)), + (TableSportsEvents.missing_subtitles != '[]')] + sports_conditions += get_exclusion_clause('sports') + missing_sports = database.execute( + select(TableSportsEvents.missing_subtitles) + .select_from(TableSportsEvents) + .join(TableSportsLeagues) + .where(reduce(operator.and_, sports_conditions))) \ + .all() + missing_sports_count = 0 + for event in missing_sports: + missing_sports_count += len(ast.literal_eval(event.missing_subtitles)) + throttled_providers = len(get_throttled_providers()) health_issues = len(get_health_issues()) @@ -71,10 +90,12 @@ def get(self): result = { "episodes": missing_episodes_count, "movies": missing_movies_count, + "sports": missing_sports_count, "providers": throttled_providers, "status": health_issues, 'sonarr_signalr': live_str if sonarr_signalr_client.connected else "DOWN", 'radarr_signalr': live_str if radarr_signalr_client.connected else "DOWN", + 'sportarr_sse': live_str if sportarr_sse_client.connected else "DOWN", 'announcements': len(get_all_announcements()), } return marshal(result, self.get_model) diff --git a/bazarr/api/sports/__init__.py b/bazarr/api/sports/__init__.py new file mode 100644 index 0000000000..0b7a4594b8 --- /dev/null +++ b/bazarr/api/sports/__init__.py @@ -0,0 +1,18 @@ +# coding=utf-8 + +from .leagues import api_ns_sports_leagues +from .events import api_ns_sports_events +from .history import api_ns_sports_history +from .tags import api_ns_sports_tags +from .wanted import api_ns_sports_wanted +from .blacklist import api_ns_sports_blacklist + + +api_ns_list_sports = [ + api_ns_sports_blacklist, + api_ns_sports_events, + api_ns_sports_history, + api_ns_sports_leagues, + api_ns_sports_tags, + api_ns_sports_wanted, +] diff --git a/bazarr/api/sports/blacklist.py b/bazarr/api/sports/blacklist.py new file mode 100644 index 0000000000..8e5960090f --- /dev/null +++ b/bazarr/api/sports/blacklist.py @@ -0,0 +1,152 @@ +# coding=utf-8 + +import pretty + +from flask_restx import Resource, Namespace, reqparse, fields, marshal + +from app.database import TableSportsEvents, TableSportsLeagues, TableBlacklistSports, database, select +from subtitles.tools.delete import delete_subtitles +from sportarr.blacklist import blacklist_log_sports, blacklist_delete_all_sports, blacklist_delete_sports +from utilities.path_mappings import path_mappings +from subtitles.mass_download import sports_event_download_subtitles +from app.event_handler import event_stream +from api.swaggerui import subtitles_language_model + +from ..utils import authenticate, postprocess + +api_ns_sports_blacklist = Namespace('Sports Blacklist', description='List, add or remove subtitles to or from ' + 'sports events blacklist') + + +@api_ns_sports_blacklist.route('sports/blacklist') +class SportsBlacklist(Resource): + get_request_parser = reqparse.RequestParser() + get_request_parser.add_argument('start', type=int, required=False, default=0, help='Paging start integer') + get_request_parser.add_argument('length', type=int, required=False, default=-1, help='Paging length integer') + + get_language_model = api_ns_sports_blacklist.model('subtitles_language_model', subtitles_language_model) + + get_response_model = api_ns_sports_blacklist.model('SportsBlacklistGetResponse', { + 'leagueTitle': fields.String(), + 'eventTitle': fields.String(), + 'partName': fields.String(), + 'sportarrLeagueId': fields.Integer(), + 'sportsEventId': fields.Integer(), + 'provider': fields.String(), + 'subs_id': fields.String(), + 'language': fields.Nested(get_language_model), + 'timestamp': fields.String(), + 'parsed_timestamp': fields.String(), + }) + + @authenticate + @api_ns_sports_blacklist.response(401, 'Not Authenticated') + @api_ns_sports_blacklist.doc(parser=get_request_parser) + def get(self): + """List blacklisted sports events subtitles""" + args = self.get_request_parser.parse_args() + start = args.get('start') + length = args.get('length') + + stmt = select(TableSportsLeagues.title.label('leagueTitle'), + TableSportsEvents.title.label('eventTitle'), + TableSportsEvents.partName, + TableBlacklistSports.sportarr_league_id.label('sportarrLeagueId'), + TableBlacklistSports.sports_event_id.label('sportsEventId'), + TableBlacklistSports.provider, + TableBlacklistSports.subs_id, + TableBlacklistSports.language, + TableBlacklistSports.timestamp) \ + .select_from(TableBlacklistSports) \ + .join(TableSportsLeagues, + onclause=TableBlacklistSports.sportarr_league_id == TableSportsLeagues.sportarrLeagueId) \ + .join(TableSportsEvents, onclause=TableBlacklistSports.sports_event_id == TableSportsEvents.id) \ + .order_by(TableBlacklistSports.timestamp.desc()) + if length > 0: + stmt = stmt.limit(length).offset(start) + + return marshal([postprocess({ + 'leagueTitle': x.leagueTitle, + 'eventTitle': x.eventTitle, + 'partName': x.partName, + 'sportarrLeagueId': x.sportarrLeagueId, + 'sportsEventId': x.sportsEventId, + 'provider': x.provider, + 'subs_id': x.subs_id, + 'language': x.language, + 'timestamp': pretty.date(x.timestamp), + 'parsed_timestamp': x.timestamp.strftime('%x %X') + }) for x in database.execute(stmt).all()], self.get_response_model, envelope='data') + + post_request_parser = reqparse.RequestParser() + post_request_parser.add_argument('leagueid', type=int, required=True, help='League ID') + post_request_parser.add_argument('eventid', type=int, required=True, help='Sports event ID') + post_request_parser.add_argument('provider', type=str, required=True, help='Provider name') + post_request_parser.add_argument('subs_id', type=str, required=True, help='Subtitles ID') + post_request_parser.add_argument('language', type=str, required=True, help='Subtitles language') + post_request_parser.add_argument('subtitles_path', type=str, required=True, help='Subtitles file path') + + @authenticate + @api_ns_sports_blacklist.doc(parser=post_request_parser) + @api_ns_sports_blacklist.response(200, 'Success') + @api_ns_sports_blacklist.response(401, 'Not Authenticated') + @api_ns_sports_blacklist.response(404, 'Sports event not found') + @api_ns_sports_blacklist.response(500, 'Subtitles file not found or permission issue.') + def post(self): + """Add a sports events subtitles to blacklist""" + args = self.post_request_parser.parse_args() + sportarr_league_id = args.get('leagueid') + sports_event_id = args.get('eventid') + provider = args.get('provider') + subs_id = args.get('subs_id') + language = args.get('language') + + eventInfo = database.execute( + select(TableSportsEvents.path) + .where(TableSportsEvents.id == sports_event_id)) \ + .first() + + if not eventInfo: + return 'Sports event not found', 404 + + media_path = eventInfo.path + subtitles_path = args.get('subtitles_path') + + blacklist_log_sports(sportarr_league_id=sportarr_league_id, + sports_event_id=sports_event_id, + provider=provider, + subs_id=subs_id, + language=language) + if delete_subtitles(media_type='sports', + language=language, + forced=False, + hi=False, + media_path=path_mappings.path_replace_sports(media_path), + subtitles_path=subtitles_path, + sportarr_league_id=sportarr_league_id, + sports_event_id=sports_event_id): + sports_event_download_subtitles(no=sports_event_id) + event_stream(type='sports-event-history') + return '', 200 + else: + return 'Subtitles file not found or permission issue.', 500 + + delete_request_parser = reqparse.RequestParser() + delete_request_parser.add_argument('all', type=str, required=False, help='Empty sports events subtitles blacklist') + delete_request_parser.add_argument('provider', type=str, required=False, help='Provider name') + delete_request_parser.add_argument('subs_id', type=str, required=False, help='Subtitles ID') + + @authenticate + @api_ns_sports_blacklist.doc(parser=delete_request_parser) + @api_ns_sports_blacklist.response(204, 'Success') + @api_ns_sports_blacklist.response(401, 'Not Authenticated') + def delete(self): + """Delete a sports events subtitles from blacklist""" + args = self.delete_request_parser.parse_args() + if args.get("all") == "true": + blacklist_delete_all_sports() + else: + provider = args.get('provider') + subs_id = args.get('subs_id') + blacklist_delete_sports(provider=provider, subs_id=subs_id) + return '', 204 diff --git a/bazarr/api/sports/events.py b/bazarr/api/sports/events.py new file mode 100644 index 0000000000..1f5fd30593 --- /dev/null +++ b/bazarr/api/sports/events.py @@ -0,0 +1,101 @@ +# coding=utf-8 + +from flask_restx import Resource, Namespace, reqparse, fields, marshal + +from app.database import TableSportsEvents, database, select +from api.swaggerui import subtitles_model, subtitles_language_model, audio_language_model + +from ..utils import authenticate, postprocess + +api_ns_sports_events = Namespace('Sports Events', description='List sports events metadata for specific leagues or ' + 'events.') + + +@api_ns_sports_events.route('sports/events') +class SportsEvents(Resource): + get_request_parser = reqparse.RequestParser() + get_request_parser.add_argument('leagueid[]', type=int, action='append', required=False, default=[], + help='League IDs to list events for') + get_request_parser.add_argument('eventid[]', type=int, action='append', required=False, default=[], + help='Sports events ID to list') + + get_subtitles_model = api_ns_sports_events.model('subtitles_model', subtitles_model) + get_subtitles_language_model = api_ns_sports_events.model('subtitles_language_model', subtitles_language_model) + get_audio_language_model = api_ns_sports_events.model('audio_language_model', audio_language_model) + + get_response_model = api_ns_sports_events.model('SportsEventGetResponse', { + 'audio_language': fields.Nested(get_audio_language_model), + 'episode': fields.Integer(), + 'missing_subtitles': fields.Nested(get_subtitles_language_model), + 'monitored': fields.Boolean(), + 'path': fields.String(), + 'season': fields.Integer(), + 'sportsEventId': fields.Integer(), + 'sportarrLeagueId': fields.Integer(), + 'subtitles': fields.Nested(get_subtitles_model), + 'title': fields.String(), + 'partName': fields.String(), + 'partNumber': fields.Integer(), + 'sceneName': fields.String(), + 'broadcastDate': fields.String(), + }) + + @authenticate + @api_ns_sports_events.doc(parser=get_request_parser) + @api_ns_sports_events.response(200, 'Success') + @api_ns_sports_events.response(401, 'Not Authenticated') + @api_ns_sports_events.response(404, 'League or Sports event ID not provided') + def get(self): + """List sports events metadata for specific leagues or events""" + args = self.get_request_parser.parse_args() + leagueId = args.get('leagueid[]') + eventId = args.get('eventid[]') + + stmt = select( + TableSportsEvents.audio_language, + TableSportsEvents.episode, + TableSportsEvents.missing_subtitles, + TableSportsEvents.monitored, + TableSportsEvents.path, + TableSportsEvents.season, + TableSportsEvents.id, + TableSportsEvents.sportarrLeagueId, + TableSportsEvents.title, + TableSportsEvents.partName, + TableSportsEvents.partNumber, + TableSportsEvents.sceneName, + TableSportsEvents.broadcastDate, + ) + + if len(eventId) > 0: + stmt_query = database.execute( + stmt + .where(TableSportsEvents.id.in_(eventId)))\ + .all() + elif len(leagueId) > 0: + # An event can hold more than one file, so order by the part as well + # to keep the prelims above the main card. + stmt_query = database.execute( + stmt + .where(TableSportsEvents.sportarrLeagueId.in_(leagueId)) + .order_by(TableSportsEvents.season.desc(), TableSportsEvents.episode.desc(), + TableSportsEvents.partNumber.asc()))\ + .all() + else: + return "League or Sports event ID not provided", 404 + + return marshal([postprocess({ + 'audio_language': x.audio_language, + 'episode': x.episode, + 'missing_subtitles': x.missing_subtitles, + 'monitored': x.monitored, + 'path': x.path, + 'season': x.season, + 'sportsEventId': x.id, + 'sportarrLeagueId': x.sportarrLeagueId, + 'title': x.title, + 'partName': x.partName, + 'partNumber': x.partNumber, + 'sceneName': x.sceneName, + 'broadcastDate': x.broadcastDate, + }) for x in stmt_query], self.get_response_model, envelope='data') diff --git a/bazarr/api/sports/history.py b/bazarr/api/sports/history.py new file mode 100644 index 0000000000..62c5d92cc2 --- /dev/null +++ b/bazarr/api/sports/history.py @@ -0,0 +1,162 @@ +# coding=utf-8 + +import operator +import ast +from functools import reduce + +from api.swaggerui import subtitles_language_model +from app.database import (TableSportsEvents, TableSportsLeagues, TableHistorySports, TableBlacklistSports, database, + select, func) + +import pretty +from flask_restx import Resource, Namespace, reqparse, fields, marshal +from ..utils import authenticate, postprocess + +api_ns_sports_history = Namespace('Sports History', description='List sports events history events') + + +@api_ns_sports_history.route('sports/history') +class SportsHistory(Resource): + get_request_parser = reqparse.RequestParser() + get_request_parser.add_argument('start', type=int, required=False, default=0, help='Paging start integer') + get_request_parser.add_argument('length', type=int, required=False, default=-1, help='Paging length integer') + get_request_parser.add_argument('eventid', type=int, required=False, help='Sports event ID') + + get_language_model = api_ns_sports_history.model('subtitles_language_model', subtitles_language_model) + + data_model = api_ns_sports_history.model('history_sports_data_model', { + 'leagueTitle': fields.String(), + 'monitored': fields.Boolean(), + 'eventTitle': fields.String(), + 'partName': fields.String(), + 'timestamp': fields.String(), + 'subs_id': fields.String(), + 'description': fields.String(), + 'sportarrLeagueId': fields.Integer(), + 'language': fields.Nested(get_language_model), + 'score': fields.String(), + 'tags': fields.List(fields.String), + 'action': fields.Integer(), + 'subtitles_path': fields.String(), + 'sportsEventId': fields.Integer(), + 'provider': fields.String(), + 'parsed_timestamp': fields.String(), + 'blacklisted': fields.Boolean(), + 'matches': fields.List(fields.String), + 'dont_matches': fields.List(fields.String), + }) + + get_response_model = api_ns_sports_history.model('SportsHistoryGetResponse', { + 'data': fields.Nested(data_model), + 'total': fields.Integer(), + }) + + @authenticate + @api_ns_sports_history.response(401, 'Not Authenticated') + @api_ns_sports_history.doc(parser=get_request_parser) + def get(self): + """List sports events history events""" + args = self.get_request_parser.parse_args() + start = args.get('start') + length = args.get('length') + eventid = args.get('eventid') + + blacklisted_subtitles = select(TableBlacklistSports.provider, + TableBlacklistSports.subs_id) \ + .subquery() + + query_conditions = [(TableSportsEvents.title.is_not(None))] + if eventid: + query_conditions.append((TableSportsEvents.id == eventid)) + + stmt = select(TableHistorySports.id, + TableSportsLeagues.title.label('leagueTitle'), + TableSportsEvents.monitored, + TableSportsEvents.title.label('eventTitle'), + TableSportsEvents.partName, + TableHistorySports.timestamp, + TableHistorySports.subs_id, + TableHistorySports.description, + TableHistorySports.sportarrLeagueId, + TableSportsEvents.path, + TableHistorySports.language, + TableHistorySports.score, + TableHistorySports.score_out_of, + TableSportsLeagues.tags, + TableHistorySports.action, + TableHistorySports.video_path, + TableHistorySports.subtitles_path, + TableHistorySports.sportsEventId, + TableHistorySports.provider, + TableSportsLeagues.sport, + TableHistorySports.matched, + TableHistorySports.not_matched, + blacklisted_subtitles.c.subs_id.label('blacklisted')) \ + .select_from(TableHistorySports) \ + .join(TableSportsLeagues, + onclause=TableHistorySports.sportarrLeagueId == TableSportsLeagues.sportarrLeagueId) \ + .join(TableSportsEvents, onclause=TableHistorySports.sportsEventId == TableSportsEvents.id) \ + .join(blacklisted_subtitles, onclause=TableHistorySports.subs_id == blacklisted_subtitles.c.subs_id, + isouter=True) \ + .where(reduce(operator.and_, query_conditions)) \ + .order_by(TableHistorySports.timestamp.desc()) + if length > 0: + stmt = stmt.limit(length).offset(start) + sports_history = [{ + 'id': x.id, + 'leagueTitle': x.leagueTitle, + 'monitored': x.monitored, + 'eventTitle': x.eventTitle, + 'partName': x.partName, + 'timestamp': x.timestamp, + 'subs_id': x.subs_id, + 'description': x.description, + 'sportarrLeagueId': x.sportarrLeagueId, + 'path': x.path, + 'language': x.language, + 'score': x.score, + 'score_out_of': x.score_out_of, + 'tags': x.tags, + 'action': x.action, + 'video_path': x.video_path, + 'subtitles_path': x.subtitles_path, + 'sportsEventId': x.sportsEventId, + 'provider': x.provider, + 'matches': x.matched, + 'dont_matches': x.not_matched, + 'blacklisted': bool(x.blacklisted), + } for x in database.execute(stmt).all()] + + for item in sports_history: + item.update(postprocess(item)) + + del item['path'] + del item['video_path'] + + if item['score']: + item['score'] = f"{round((int(item['score']) * 100 / item['score_out_of']), 2)}%" + + # Make timestamp pretty + if item['timestamp']: + item["parsed_timestamp"] = item['timestamp'].strftime('%x %X') + item['timestamp'] = pretty.date(item["timestamp"]) + + # Parse matches and dont_matches + if item['matches']: + item.update({'matches': ast.literal_eval(item['matches'])}) + else: + item.update({'matches': []}) + + if item['dont_matches']: + item.update({'dont_matches': ast.literal_eval(item['dont_matches'])}) + else: + item.update({'dont_matches': []}) + + count = database.execute( + select(func.count()) + .select_from(TableHistorySports) + .join(TableSportsEvents, onclause=TableHistorySports.sportsEventId == TableSportsEvents.id) + .where(TableSportsEvents.title.is_not(None))) \ + .scalar() + + return marshal({'data': sports_history, 'total': count}, self.get_response_model) diff --git a/bazarr/api/sports/leagues.py b/bazarr/api/sports/leagues.py new file mode 100644 index 0000000000..8bc75c0dcf --- /dev/null +++ b/bazarr/api/sports/leagues.py @@ -0,0 +1,215 @@ +# coding=utf-8 + +import operator + +from flask_restx import Resource, Namespace, reqparse, fields, marshal +from functools import reduce + +from app.database import get_exclusion_clause, TableSportsEvents, TableSportsLeagues, database, select, update, func +from subtitles.indexer.sports import list_missing_subtitles_sports, sports_scan_subtitles +from subtitles.mass_download import league_download_subtitles +from subtitles.wanted import wanted_search_missing_subtitles_sports +from sportarr.sync.leagues import update_one_league +from app.event_handler import event_stream +from api.swaggerui import subtitles_language_model + +from ..utils import authenticate, None_Keys, postprocess + +api_ns_sports_leagues = Namespace('Sports Leagues', description='List sports leagues metadata, update languages ' + 'profile or run actions on specific leagues.') + + +@api_ns_sports_leagues.route('sports/leagues') +class SportsLeagues(Resource): + get_request_parser = reqparse.RequestParser() + get_request_parser.add_argument('start', type=int, required=False, default=0, help='Paging start integer') + get_request_parser.add_argument('length', type=int, required=False, default=-1, help='Paging length integer') + get_request_parser.add_argument('leagueid[]', type=int, action='append', required=False, default=[], + help='Sportarr league IDs to get') + + get_subtitles_language_model = api_ns_sports_leagues.model('subtitles_language_model', subtitles_language_model) + + data_model = api_ns_sports_leagues.model('sports_leagues_data_model', { + 'sportarrLeagueId': fields.Integer(), + 'externalId': fields.String(), + 'title': fields.String(), + 'sortTitle': fields.String(), + 'sport': fields.String(), + 'path': fields.String(), + 'poster': fields.String(), + 'fanart': fields.String(), + 'overview': fields.String(), + 'monitored': fields.Boolean(), + 'profileId': fields.Integer(), + 'tags': fields.List(fields.String), + 'audio_language': fields.Nested(get_subtitles_language_model), + 'episodeFileCount': fields.Integer(), + 'episodeMissingCount': fields.Integer(), + }) + + get_response_model = api_ns_sports_leagues.model('SportsLeaguesGetResponse', { + 'data': fields.Nested(data_model), + 'total': fields.Integer(), + }) + + @authenticate + @api_ns_sports_leagues.response(401, 'Not Authenticated') + @api_ns_sports_leagues.doc(parser=get_request_parser) + def get(self): + """List sports leagues metadata""" + args = self.get_request_parser.parse_args() + start = args.get('start') + length = args.get('length') + leagueId = args.get('leagueid[]') + + stmt = select(TableSportsLeagues.sportarrLeagueId, + TableSportsLeagues.externalId, + TableSportsLeagues.title, + TableSportsLeagues.sortTitle, + TableSportsLeagues.sport, + TableSportsLeagues.path, + TableSportsLeagues.poster, + TableSportsLeagues.fanart, + TableSportsLeagues.overview, + TableSportsLeagues.monitored, + TableSportsLeagues.profileId, + TableSportsLeagues.tags, + TableSportsLeagues.audio_language) \ + .order_by(TableSportsLeagues.sortTitle) + + if len(leagueId) > 0: + stmt = stmt.where(TableSportsLeagues.sportarrLeagueId.in_(leagueId)) + elif length > 0: + stmt = stmt.limit(length).offset(start) + + results = [] + for x in database.execute(stmt).all(): + # Counted per playable file, because a row is one part. An event with + # two parts needs subtitles for both. + file_count = database.execute( + select(func.count()) + .select_from(TableSportsEvents) + .where(TableSportsEvents.sportarrLeagueId == x.sportarrLeagueId)) \ + .scalar() + + missing_conditions = [(TableSportsEvents.sportarrLeagueId == x.sportarrLeagueId), + (TableSportsEvents.missing_subtitles.is_not(None)), + (TableSportsEvents.missing_subtitles != '[]')] + missing_conditions += get_exclusion_clause('sports') + missing_count = database.execute( + select(func.count()) + .select_from(TableSportsEvents) + .join(TableSportsLeagues) + .where(reduce(operator.and_, missing_conditions))) \ + .scalar() + + results.append(postprocess({ + 'sportarrLeagueId': x.sportarrLeagueId, + 'externalId': x.externalId, + 'title': x.title, + 'sortTitle': x.sortTitle, + 'sport': x.sport, + 'path': x.path, + 'poster': x.poster, + 'fanart': x.fanart, + 'overview': x.overview, + 'monitored': x.monitored, + 'profileId': x.profileId, + 'tags': x.tags, + 'audio_language': x.audio_language, + 'episodeFileCount': file_count, + 'episodeMissingCount': missing_count, + })) + + count = database.execute( + select(func.count()) + .select_from(TableSportsLeagues)) \ + .scalar() + + return marshal({'data': results, 'total': count}, self.get_response_model) + + post_request_parser = reqparse.RequestParser() + post_request_parser.add_argument('leagueid', type=int, action='append', required=False, default=[], + help='Sportarr league ID') + post_request_parser.add_argument('profileid', type=str, action='append', required=False, default=[], + help='Languages profile(s) ID or "none"') + + @authenticate + @api_ns_sports_leagues.doc(parser=post_request_parser) + @api_ns_sports_leagues.response(204, 'Success') + @api_ns_sports_leagues.response(401, 'Not Authenticated') + @api_ns_sports_leagues.response(404, 'Languages profile not found') + def post(self): + """Update specific sports leagues languages profile""" + args = self.post_request_parser.parse_args() + leagueIdList = args.get('leagueid') + profileIdList = args.get('profileid') + + for idx in range(len(leagueIdList)): + leagueId = leagueIdList[idx] + profileId = profileIdList[idx] + + if profileId in None_Keys: + profileId = None + else: + try: + profileId = int(profileId) + except Exception: + return 'Languages profile not found', 404 + + database.execute( + update(TableSportsLeagues) + .values(profileId=profileId) + .where(TableSportsLeagues.sportarrLeagueId == leagueId)) + + list_missing_subtitles_sports(no=leagueId) + + event_stream(type='sports-league', payload=leagueId) + + event_id_list = database.execute( + select(TableSportsEvents.id) + .where(TableSportsEvents.sportarrLeagueId == leagueId))\ + .all() + + for item in event_id_list: + event_stream(type='sports-event-wanted', payload=item.id) + + event_stream(type='badges') + + return '', 204 + + patch_request_parser = reqparse.RequestParser() + patch_request_parser.add_argument('leagueid', type=int, required=False, help='Sportarr league ID') + patch_request_parser.add_argument('action', type=str, required=False, + help='Action to perform from ["scan-disk", "search-missing", "search-wanted", ' + '"sync"]') + + @authenticate + @api_ns_sports_leagues.doc(parser=patch_request_parser) + @api_ns_sports_leagues.response(204, 'Success') + @api_ns_sports_leagues.response(400, 'Unknown action') + @api_ns_sports_leagues.response(401, 'Not Authenticated') + @api_ns_sports_leagues.response(500, 'League directory not found. Path mapping issue?') + def patch(self): + """Run actions on specific sports leagues""" + args = self.patch_request_parser.parse_args() + leagueid = args.get('leagueid') + action = args.get('action') + if action == "scan-disk": + sports_scan_subtitles(leagueid) + return '', 204 + elif action == "search-missing": + try: + league_download_subtitles(leagueid) + except OSError: + return 'League directory not found. Path mapping issue?', 500 + else: + return '', 204 + elif action == "search-wanted": + wanted_search_missing_subtitles_sports() + return '', 204 + elif action == "sync": + update_one_league(leagueid, 'updated') + return '', 204 + + return 'Unknown action', 400 diff --git a/bazarr/api/sports/tags.py b/bazarr/api/sports/tags.py new file mode 100644 index 0000000000..f3d03ab104 --- /dev/null +++ b/bazarr/api/sports/tags.py @@ -0,0 +1,31 @@ +# coding=utf-8 + +import ast + +from flask_restx import Resource, Namespace, fields, marshal + +from app.database import TableSportsLeagues, database, select + +from ..utils import authenticate + +api_ns_sports_tags = Namespace('Sports Tags', description='List tags assigned to sports leagues') + + +@api_ns_sports_tags.route('sports/leagues/tags') +class SportsLeaguesTags(Resource): + get_response_model = api_ns_sports_tags.model('SportsLeaguesTagsGetResponse', { + 'tag': fields.String(), + }) + + @authenticate + @api_ns_sports_tags.response(200, 'Success') + @api_ns_sports_tags.response(401, 'Not Authenticated') + def get(self): + """List all distinct tags assigned to any sports league""" + rows = database.execute( + select(TableSportsLeagues.tags).where(TableSportsLeagues.tags.is_not(None))).all() + tags = set() + for row in rows: + if row.tags: + tags.update(ast.literal_eval(row.tags)) + return marshal([{'tag': tag} for tag in sorted(tags)], self.get_response_model, envelope='data') diff --git a/bazarr/api/sports/wanted.py b/bazarr/api/sports/wanted.py new file mode 100644 index 0000000000..b297841fad --- /dev/null +++ b/bazarr/api/sports/wanted.py @@ -0,0 +1,99 @@ +# coding=utf-8 + +import operator + +from flask_restx import Resource, Namespace, reqparse, fields, marshal +from functools import reduce + +from app.database import get_exclusion_clause, TableSportsEvents, TableSportsLeagues, database, select, func +from api.swaggerui import subtitles_language_model + +from ..utils import authenticate, postprocess + +api_ns_sports_wanted = Namespace('Sports Wanted', description='List sports events wanted subtitles') + + +@api_ns_sports_wanted.route('sports/wanted') +class SportsWanted(Resource): + get_request_parser = reqparse.RequestParser() + get_request_parser.add_argument('start', type=int, required=False, default=0, help='Paging start integer') + get_request_parser.add_argument('length', type=int, required=False, default=-1, help='Paging length integer') + get_request_parser.add_argument('eventid[]', type=int, action='append', required=False, default=[], + help='Sports events ID to list') + + get_subtitles_language_model = api_ns_sports_wanted.model('subtitles_language_model', subtitles_language_model) + + data_model = api_ns_sports_wanted.model('wanted_sports_data_model', { + 'leagueTitle': fields.String(), + 'eventTitle': fields.String(), + 'partName': fields.String(), + 'missing_subtitles': fields.Nested(get_subtitles_language_model), + 'sportarrLeagueId': fields.Integer(), + 'sportsEventId': fields.Integer(), + 'sceneName': fields.String(), + 'tags': fields.List(fields.String), + 'sport': fields.String(), + }) + + get_response_model = api_ns_sports_wanted.model('SportsWantedGetResponse', { + 'data': fields.Nested(data_model), + 'total': fields.Integer(), + }) + + @authenticate + @api_ns_sports_wanted.response(401, 'Not Authenticated') + @api_ns_sports_wanted.doc(parser=get_request_parser) + def get(self): + """List sports events wanted subtitles""" + args = self.get_request_parser.parse_args() + eventid = args.get('eventid[]') + + wanted_conditions = [(TableSportsEvents.missing_subtitles.is_not(None)), + (TableSportsEvents.missing_subtitles != '[]')] + if len(eventid) > 0: + wanted_conditions.append((TableSportsEvents.id.in_(eventid))) + start = 0 + length = 0 + else: + start = args.get('start') + length = args.get('length') + + wanted_conditions += get_exclusion_clause('sports') + wanted_condition = reduce(operator.and_, wanted_conditions) + + stmt = select(TableSportsLeagues.title.label('leagueTitle'), + TableSportsEvents.title.label('eventTitle'), + TableSportsEvents.partName, + TableSportsEvents.missing_subtitles, + TableSportsEvents.sportarrLeagueId, + TableSportsEvents.id.label('sportsEventId'), + TableSportsEvents.sceneName, + TableSportsLeagues.tags, + TableSportsLeagues.sport) \ + .select_from(TableSportsEvents) \ + .join(TableSportsLeagues) \ + .where(wanted_condition) + + if length > 0: + stmt = stmt.order_by(TableSportsEvents.id.desc()).limit(length).offset(start) + + results = [postprocess({ + 'leagueTitle': x.leagueTitle, + 'eventTitle': x.eventTitle, + 'partName': x.partName, + 'missing_subtitles': x.missing_subtitles, + 'sportarrLeagueId': x.sportarrLeagueId, + 'sportsEventId': x.sportsEventId, + 'sceneName': x.sceneName, + 'tags': x.tags, + 'sport': x.sport, + }) for x in database.execute(stmt).all()] + + count = database.execute( + select(func.count()) + .select_from(TableSportsEvents) + .join(TableSportsLeagues) + .where(wanted_condition)) \ + .scalar() + + return marshal({'data': results, 'total': count}, self.get_response_model) diff --git a/bazarr/api/utils.py b/bazarr/api/utils.py index a5ca0f60b3..4182c2cda7 100644 --- a/bazarr/api/utils.py +++ b/bazarr/api/utils.py @@ -232,12 +232,20 @@ def postprocess(item): item['external_subtitles'] = path_replace(item['external_subtitles']) # map poster and fanart to server proxy + # Sonarr and Radarr give a path to proxy. Sportarr gives a complete URL, + # which needs no proxy and must not get a prefix. if item.get('poster') is not None: poster = item['poster'] - item['poster'] = f"{base_url}/images/{'movies' if item.get('radarrId') else 'series'}{poster}" if poster else None + item['poster'] = _proxied_image(poster, base_url, item) if poster else None if item.get('fanart') is not None: fanart = item['fanart'] - item['fanart'] = f"{base_url}/images/{'movies' if item.get('radarrId') else 'series'}{fanart}" if fanart else None + item['fanart'] = _proxied_image(fanart, base_url, item) if fanart else None return item + + +def _proxied_image(image, base_url, item): + if image.startswith(('http://', 'https://')): + return image + return f"{base_url}/images/{'movies' if item.get('radarrId') else 'series'}{image}" diff --git a/bazarr/app/config.py b/bazarr/app/config.py index 8dc3cd4366..f404f11c39 100644 --- a/bazarr/app/config.py +++ b/bazarr/app/config.py @@ -708,6 +708,7 @@ def save_settings(settings_items): update_schedule = False sonarr_changed = False radarr_changed = False + sportarr_changed = False update_path_map = False configure_proxy = False configure_ssl_verify = False @@ -808,6 +809,10 @@ def save_settings(settings_items): 'settings-sonarr-base_url', 'settings-sonarr-ssl', 'settings-sonarr-apikey']: sonarr_changed = True + if key in ['settings-general-use_sportarr', 'settings-sportarr-ip', 'settings-sportarr-port', + 'settings-sportarr-base_url', 'settings-sportarr-ssl', 'settings-sportarr-apikey']: + sportarr_changed = True + if key in ['settings-general-use_radarr', 'settings-radarr-ip', 'settings-radarr-port', 'settings-radarr-base_url', 'settings-radarr-ssl', 'settings-radarr-apikey']: radarr_changed = True @@ -979,6 +984,13 @@ def save_settings(settings_items): except Exception: pass + if sportarr_changed: + from sportarr.sse_client import sportarr_sse_client + try: + sportarr_sse_client.restart() + except Exception: + pass + if update_path_map: from utilities.path_mappings import path_mappings path_mappings.update() diff --git a/bazarr/app/database.py b/bazarr/app/database.py index 201f1c9ca1..8b29893f60 100644 --- a/bazarr/app/database.py +++ b/bazarr/app/database.py @@ -502,6 +502,42 @@ class TableSportsEventsSubtitles(Base): ) +class TableBlacklistSports(Base): + __tablename__ = 'table_blacklist_sports' + + id = mapped_column(Integer, primary_key=True) + language = mapped_column(Text) + provider = mapped_column(Text) + # Points at the event row, which is one part, not at the Sportarr event id. + sports_event_id = mapped_column(Integer, ForeignKey('table_sports_events.id', ondelete='CASCADE')) + sportarr_league_id = mapped_column(Integer, ForeignKey('table_sports_leagues.sportarrLeagueId', + ondelete='CASCADE')) + subs_id = mapped_column(Text) + timestamp = mapped_column(DateTime, default=datetime.now) + + +class TableHistorySports(Base): + __tablename__ = 'table_history_sports' + + id = mapped_column(Integer, primary_key=True) + action = mapped_column(Integer, nullable=False) + description = mapped_column(Text, nullable=False) + language = mapped_column(Text) + provider = mapped_column(Text) + score = mapped_column(Integer) + score_out_of = mapped_column(Integer, nullable=True) + # Points at the event row, which is one part, not at the Sportarr event id. + sportsEventId = mapped_column(Integer, ForeignKey('table_sports_events.id', ondelete='CASCADE')) + sportarrLeagueId = mapped_column(Integer, ForeignKey('table_sports_leagues.sportarrLeagueId', ondelete='CASCADE')) + subs_id = mapped_column(Text) + subtitles_path = mapped_column(Text) + timestamp = mapped_column(DateTime, nullable=False, default=datetime.now) + video_path = mapped_column(Text) + matched = mapped_column(Text) + not_matched = mapped_column(Text) + upgradedFromId = mapped_column(Integer, ForeignKey('table_history_sports.id')) + + class TableSportsLeaguesRootfolder(Base): __tablename__ = 'table_sports_leagues_rootfolder' @@ -574,6 +610,10 @@ def get_exclusion_clause(exclusion_type): tagsList = settings.sonarr.excluded_tags for tag in tagsList: where_clause.append(~(TableShows.tags.contains(f"\'{tag}\'"))) + elif exclusion_type == 'sports': + tagsList = settings.sportarr.excluded_tags + for tag in tagsList: + where_clause.append(~(TableSportsLeagues.tags.contains(f"\'{tag}\'"))) else: tagsList = settings.radarr.excluded_tags for tag in tagsList: @@ -584,6 +624,11 @@ def get_exclusion_clause(exclusion_type): if monitoredOnly: where_clause.append((TableEpisodes.monitored == 'True')) # noqa E712 where_clause.append((TableShows.monitored == 'True')) # noqa E712 + elif exclusion_type == 'sports': + monitoredOnly = settings.sportarr.only_monitored + if monitoredOnly: + where_clause.append((TableSportsEvents.monitored == 'True')) # noqa E712 + where_clause.append((TableSportsLeagues.monitored == 'True')) # noqa E712 else: monitoredOnly = settings.radarr.only_monitored if monitoredOnly: @@ -597,6 +642,10 @@ def get_exclusion_clause(exclusion_type): exclude_season_zero = settings.sonarr.exclude_season_zero if exclude_season_zero: where_clause.append((TableEpisodes.season != 0)) + elif exclusion_type == 'sports': + # Sport is the closest a league has to a series type. + for item in settings.sportarr.excluded_sports: + where_clause.append((TableSportsLeagues.sport != item)) return where_clause @@ -797,6 +846,48 @@ def fix_languages_profiles_with_duplicate_ids(): ) +def get_sports_subtitles(sports_event_id: int) -> List[dict]: + """ + Retrieves the subtitles indexed for one sports event row. + + A row is one playable file, so a card's prelims and main card each have + their own subtitles rather than sharing a set. + + :param sports_event_id: The table_sports_events row id. + :type sports_event_id: int + :return: A list of dictionaries with the same shape get_subtitles returns. + :rtype: List[dict] + """ + from languages.get_languages import alpha3_from_alpha2, language_from_alpha2 + + subtitles = [] + events_subtitles = database.execute( + select(TableSportsEventsSubtitles.path, + TableSportsEventsSubtitles.language, + TableSportsEventsSubtitles.forced, + TableSportsEventsSubtitles.hi, + TableSportsEventsSubtitles.size, + TableSportsEventsSubtitles.embedded_track_id, + TableSportsEventsSubtitles.id) + .where(TableSportsEventsSubtitles.sportsEventId == sports_event_id) + ).all() + + for event_subtitles in events_subtitles: + subtitles.append( + {"path": path_mappings.path_replace_sports(event_subtitles.path), + "name": language_from_alpha2(event_subtitles.language), + "code2": event_subtitles.language, + "code3": alpha3_from_alpha2(event_subtitles.language), + "forced": event_subtitles.forced, + "hi": event_subtitles.hi, + "file_size": event_subtitles.size, + "embedded_track_id": event_subtitles.embedded_track_id, + "id": event_subtitles.id} + ) + + return subtitles + + def get_subtitles(sonarr_episode_id: int = None, radarr_id: int = None) -> List[dict]: """ Retrieves a list of subtitles based on the provided episode or movie identifiers. diff --git a/bazarr/app/scheduler.py b/bazarr/app/scheduler.py index 7c41e1ddb5..4dd6a37d77 100644 --- a/bazarr/app/scheduler.py +++ b/bazarr/app/scheduler.py @@ -24,7 +24,9 @@ from radarr.sync.movies import update_movies from subtitles.indexer.movies import movies_full_scan_subtitles from subtitles.indexer.series import series_full_scan_subtitles -from subtitles.wanted import wanted_search_missing_subtitles_series, wanted_search_missing_subtitles_movies +from subtitles.indexer.sports import sports_full_scan_subtitles +from subtitles.wanted import wanted_search_missing_subtitles_series, wanted_search_missing_subtitles_movies, \ + wanted_search_missing_subtitles_sports from subtitles.upgrade import upgrade_subtitles from utilities.cache import cache_maintenance from utilities.health import check_health @@ -107,6 +109,7 @@ def update_configurable_tasks(self): self.__sportarr_update_task() self.__sonarr_full_update_task() self.__radarr_full_update_task() + self.__sportarr_full_update_task() self.__update_bazarr_task() self.__search_wanted_subtitles_task() self.__upgrade_subtitles_task() @@ -285,6 +288,27 @@ def __radarr_full_update_task(self): misfire_grace_time=15, id='movies_full_scan_subtitles', name='Index All Existing Movies Subtitles', replace_existing=True, kwargs=dict(wait_for_completion=True)) + def __sportarr_full_update_task(self): + if settings.general.use_sportarr: + full_update = settings.sportarr.full_update + if full_update == "Daily": + self.aps_scheduler.add_job( + sports_full_scan_subtitles, 'cron', hour=settings.sportarr.full_update_hour, max_instances=1, + coalesce=True, misfire_grace_time=15, id='sports_full_scan_subtitles', + name='Index All Existing Sports Events Subtitles', replace_existing=True, + kwargs=dict(wait_for_completion=True)) + elif full_update == "Weekly": + self.aps_scheduler.add_job( + sports_full_scan_subtitles, 'cron', day_of_week=settings.sportarr.full_update_day, + hour=settings.sportarr.full_update_hour, max_instances=1, coalesce=True, misfire_grace_time=15, + id='sports_full_scan_subtitles', name='Index All Existing Sports Events Subtitles', + replace_existing=True, kwargs=dict(wait_for_completion=True)) + elif full_update == "Manually": + self.aps_scheduler.add_job( + sports_full_scan_subtitles, 'cron', year=in_a_century(), max_instances=1, coalesce=True, + misfire_grace_time=15, id='sports_full_scan_subtitles', kwargs=dict(wait_for_completion=True), + name='Index All Existing Sports Events Subtitles', replace_existing=True) + def __update_bazarr_task(self): if not args.no_update and os.environ["BAZARR_VERSION"] not in ['', 'unknown']: task_name = 'Update Bazarr' @@ -327,6 +351,15 @@ def __search_wanted_subtitles_task(self): misfire_grace_time=15, id='wanted_search_missing_subtitles_movies', name='Search for Missing Movies Subtitles', replace_existing=True, kwargs=dict(wait_for_completion=True)) + if settings.general.use_sportarr: + # A sports event is scored as an episode, so it follows the series + # search frequency rather than the movies one. + self.aps_scheduler.add_job( + wanted_search_missing_subtitles_sports, 'interval', + hours=int(settings.general.wanted_search_frequency), max_instances=1, coalesce=True, + misfire_grace_time=15, id='wanted_search_missing_subtitles_sports', + name='Search for Missing Sports Events Subtitles', replace_existing=True, + kwargs=dict(wait_for_completion=True)) def __upgrade_subtitles_task(self): if settings.general.use_sonarr or settings.general.use_radarr: diff --git a/bazarr/main.py b/bazarr/main.py index 9e75e7d23b..7fefa81420 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -46,6 +46,7 @@ from languages.get_languages import load_language_in_db # noqa E402 from app.jobs_queue import jobs_queue # noqa E402 from app.signalr_client import sonarr_signalr_client, radarr_signalr_client # noqa E402 +from sportarr.sse_client import sportarr_sse_client # noqa E402 from app.server import webserver, app # noqa E402 from app.announcements import get_announcements_to_file # noqa E402 from utilities.central import stop_bazarr # noqa E402 @@ -94,6 +95,10 @@ radarr_signalr_thread = Thread(target=radarr_signalr_client.start) radarr_signalr_thread.daemon = True radarr_signalr_thread.start() + if settings.general.use_sportarr: + sportarr_sse_thread = Thread(target=sportarr_sse_client.start) + sportarr_sse_thread.daemon = True + sportarr_sse_thread.start() if __name__ == "__main__": diff --git a/bazarr/sportarr/blacklist.py b/bazarr/sportarr/blacklist.py new file mode 100644 index 0000000000..65dae4a199 --- /dev/null +++ b/bazarr/sportarr/blacklist.py @@ -0,0 +1,39 @@ +# coding=utf-8 + +from datetime import datetime + +from app.database import TableBlacklistSports, database, insert, delete, select +from app.event_handler import event_stream + + +def get_blacklist_sports(): + return [(item.provider, item.subs_id) for item in + database.execute( + select(TableBlacklistSports.provider, TableBlacklistSports.subs_id)) + .all()] + + +def blacklist_log_sports(sportarr_league_id, sports_event_id, provider, subs_id, language): + database.execute( + insert(TableBlacklistSports) + .values( + sportarr_league_id=sportarr_league_id, + sports_event_id=sports_event_id, + timestamp=datetime.now(), + provider=provider, + subs_id=subs_id, + language=language + )) + event_stream(type='sports-event-blacklist') + + +def blacklist_delete_sports(provider, subs_id): + database.execute( + delete(TableBlacklistSports) + .where((TableBlacklistSports.provider == provider) & (TableBlacklistSports.subs_id == subs_id))) + event_stream(type='sports-event-blacklist', action='delete') + + +def blacklist_delete_all_sports(): + database.execute(delete(TableBlacklistSports)) + event_stream(type='sports-event-blacklist', action='delete') diff --git a/bazarr/sportarr/history.py b/bazarr/sportarr/history.py new file mode 100644 index 0000000000..ea1d37b9eb --- /dev/null +++ b/bazarr/sportarr/history.py @@ -0,0 +1,44 @@ +# coding=utf-8 + +from datetime import datetime + +from subliminal_patch.score import MAX_SCORES + +from app.database import TableHistorySports, database, insert +from app.event_handler import event_stream + + +def history_log_sports(action, sportarr_league_id, sports_event_id, result, fake_provider=None, fake_score=None, + upgraded_from_id=None): + description = result.message + video_path = result.path + language = result.language_code + provider = fake_provider or result.provider + score = fake_score or result.score + subs_id = result.subs_id + subtitles_path = result.subs_path + matched = result.matched + not_matched = result.not_matched + + database.execute( + insert(TableHistorySports) + .values( + action=action, + sportarrLeagueId=sportarr_league_id, + sportsEventId=sports_event_id, + timestamp=datetime.now(), + description=description, + video_path=video_path, + language=language, + provider=provider, + score=score, + # A sports event is scored as an episode. It has a season and an + # episode number from Sportarr, so the episode scores apply. + score_out_of=MAX_SCORES['episode'] if score else None, + subs_id=subs_id, + subtitles_path=subtitles_path, + matched=str(matched) if matched else None, + not_matched=str(not_matched) if not_matched else None, + upgradedFromId=upgraded_from_id, + )) + event_stream(type='sports-event-history') diff --git a/bazarr/sportarr/notify.py b/bazarr/sportarr/notify.py new file mode 100644 index 0000000000..13bfb38ed1 --- /dev/null +++ b/bazarr/sportarr/notify.py @@ -0,0 +1,18 @@ +# coding=utf-8 + +import logging +import requests + +from app.config import settings +from sportarr.info import url_api_sportarr +from constants import HEADERS + + +def notify_sportarr(sportarr_league_id): + # Sportarr rescans the league folder to find a file that changed outside of + # it, a subtitle for example. There is no per-event equivalent. + try: + url = f"{url_api_sportarr()}leagues/{int(sportarr_league_id)}/scan?apikey={settings.sportarr.apikey}" + requests.post(url, timeout=int(settings.sportarr.http_timeout), verify=False, headers=HEADERS) + except Exception: + logging.exception('BAZARR cannot notify Sportarr') diff --git a/bazarr/sportarr/sse_client.py b/bazarr/sportarr/sse_client.py new file mode 100644 index 0000000000..9c5515c32c --- /dev/null +++ b/bazarr/sportarr/sse_client.py @@ -0,0 +1,144 @@ +# coding=utf-8 + +import json +import logging +import time + +from requests import Session +from requests.exceptions import RequestException + +from app.config import settings +from app.database import database, select, TableSportsEvents +from constants import HEADERS +from sportarr.info import url_sportarr +from sportarr.sync.events import sync_events +from sportarr.sync.utils import get_event_from_sportarr_api +from sportarr.sync.leagues import update_one_league + + +class SportarrSSEClient: + """Reads Sportarr's event stream so changes arrive without waiting for a sync. + + Sportarr sends Server-Sent Events rather than SignalR. Each frame carries a + monotonic id, so a reconnect asks for everything after the last id seen and + resumes exactly where it stopped. There is no full resync after a drop. + """ + + def __init__(self): + self.session = Session() + self.session.verify = False + self.session.headers = HEADERS + self.connected = False + self.stopped = False + self.last_event_id = None + + def start(self): + if not settings.general.use_sportarr: + return + + self.stopped = False + while not self.stopped: + try: + self.connect() + except RequestException: + logging.debug('BAZARR connection to Sportarr event stream was lost.') + except Exception: + logging.exception('BAZARR unexpected error reading the Sportarr event stream.') + + if self.stopped: + break + + self.connected = False + # Sportarr keeps a window of past events, so reconnecting after a + # pause still returns what was missed. + time.sleep(5) + + def stop(self): + self.stopped = True + self.connected = False + logging.info('BAZARR SSE client for Sportarr is now disconnected.') + + def restart(self): + if self.connected: + self.stop() + if settings.general.use_sportarr: + self.start() + + def connect(self): + url = f"{url_sportarr()}/api/stream?apikey={settings.sportarr.apikey}" + if self.last_event_id: + url += f"&since={self.last_event_id}" + + logging.info('BAZARR trying to connect to Sportarr event stream...') + with self.session.get(url, stream=True, timeout=(10, None)) as response: + response.raise_for_status() + self.connected = True + logging.info('BAZARR SSE client for Sportarr is connected and waiting for events.') + + event_id = None + for line in response.iter_lines(decode_unicode=True): + if self.stopped: + break + if not line: + continue + # Lines opening with a colon are keepalive comments. + if line.startswith(':'): + continue + + if line.startswith('id:'): + event_id = line[3:].strip() + elif line.startswith('data:'): + if event_id: + self.last_event_id = event_id + self.dispatch(line[5:].strip()) + + def dispatch(self, raw): + try: + payload = json.loads(raw) + except ValueError: + logging.debug('BAZARR could not read a frame from the Sportarr event stream.') + return + + resource = payload.get('resourceType') + action = payload.get('action') + event_id = payload.get('eventId') + league_id = payload.get('leagueId') + + if resource == 'league': + if league_id: + update_one_league(league_id, action='deleted' if action == 'removed' else 'updated') + return + + if resource not in ('event', 'file'): + return + + # Every frame names its event, so only that event's league is resynced + # rather than the whole catalogue. + if not league_id and event_id: + league_id = self.league_id_for_event(event_id) + + if not league_id: + logging.debug('BAZARR received a Sportarr event with no league to sync.') + return + + sync_events(league_id=league_id) + + @staticmethod + def league_id_for_event(event_id): + """Find the league for an event a frame did not name. + + A file frame carries only the event, so the league comes from a row we + already hold. On a first run there are no rows yet, so Sportarr is + asked directly rather than dropping the frame. + """ + row = database.execute( + select(TableSportsEvents.sportarrLeagueId) + .where(TableSportsEvents.sportarrEventId == event_id)).first() + if row: + return row[0] + + event = get_event_from_sportarr_api(apikey_sportarr=settings.sportarr.apikey, event_id=event_id) + return event.get('leagueId') if event else None + + +sportarr_sse_client = SportarrSSEClient() diff --git a/bazarr/sportarr/sync/events.py b/bazarr/sportarr/sync/events.py new file mode 100644 index 0000000000..fede4470b7 --- /dev/null +++ b/bazarr/sportarr/sync/events.py @@ -0,0 +1,182 @@ +# coding=utf-8 + +import logging + +from sqlalchemy.exc import IntegrityError +from datetime import datetime + +from constants import MINIMUM_VIDEO_SIZE +from app.database import database, TableSportsLeagues, TableSportsEvents, delete, update, insert, select +from app.config import settings +from utilities.helper import bool_map +from utilities.path_mappings import path_mappings +from app.event_handler import event_stream +from subtitles.indexer.sports import store_subtitles_sports + +from .parser import eventParser +from .utils import get_events_from_sportarr_api + +FEATURE_PREFIX = "SYNC_EVENTS " + + +def trace(message): + if settings.general.debug: + logging.debug(FEATURE_PREFIX + message) + + +def get_event_row_id(sportarr_event_id, part_number): + # The row id is not known until the insert runs, and the indexer needs it. + row = database.execute( + select(TableSportsEvents.id) + .where(TableSportsEvents.sportarrEventId == sportarr_event_id) + .where(TableSportsEvents.partNumber == part_number)).first() + return row[0] if row else None + + +def get_events_monitored_table(league_id): + events_monitored = database.execute( + select(TableSportsEvents.sportarrEventId, TableSportsEvents.monitored) + .where(TableSportsEvents.sportarrLeagueId == league_id))\ + .all() + return dict((x, y) for x, y in events_monitored) + + +def sync_events(league_id): + logging.debug(f'BAZARR Starting events sync from Sportarr for league ID {league_id}.') + apikey_sportarr = settings.sportarr.apikey + + if not league_id: + return + + # Current rows for this league, keyed by the event and its part + current_events_in_db_row_as_dict = { + (row[0].sportarrEventId, row[0].partNumber): row[0].to_dict() + for row in database.execute( + select(TableSportsEvents) + .where(TableSportsEvents.sportarrLeagueId == league_id)) + .all()} + current_events_id_db_list = list(current_events_in_db_row_as_dict) + + current_events_sportarr = [] + events_to_update = [] + events_to_add = [] + + events = get_events_from_sportarr_api(apikey_sportarr=apikey_sportarr, league_id=league_id) + if events is None: + # The walk failed partway, so nothing can be said about what still + # exists. Returning stops a failed request being read as a league + # that lost its events. + logging.debug(f'BAZARR could not read every event page for league {league_id}, so nothing is removed.') + return + + sync_monitored = settings.sportarr.sync_only_monitored_leagues and settings.sportarr.sync_only_monitored_events + events_monitored = get_events_monitored_table(league_id) if sync_monitored else None + skipped_count = 0 + + for event in events: + if not event.get('hasFile'): + continue + + if sync_monitored: + try: + monitored_status_db = bool_map[events_monitored[event['id']]] + except KeyError: + monitored_status_db = None + + if monitored_status_db is None: + # not in db, might need to add, if we have a file on disk + pass + elif monitored_status_db != event['monitored']: + # monitored status changed and we don't know about it until now + trace(f"(Monitor Status Mismatch) {event['title']}") + elif not event['monitored']: + # Keep unmonitored events in the seen list, otherwise they are + # deleted from the database + for existing_file in event.get('files') or []: + current_events_sportarr.append((event['id'], existing_file.get('partNumber') or 0)) + skipped_count += 1 + continue + + # One row per playable file. A card ships prelims and a main card + # separately, and each needs its own subtitles. + for file in event.get('files') or []: + if not file.get('filePath'): + continue + if (file.get('size') or 0) <= MINIMUM_VIDEO_SIZE: + continue + + key = (event['id'], file.get('partNumber') or 0) + current_events_sportarr.append(key) + + parsed_event = eventParser(event, file) + if key in current_events_in_db_row_as_dict: + if not set(parsed_event.items()).issubset(set(current_events_in_db_row_as_dict[key].items())): + events_to_update.append(parsed_event) + else: + events_to_add.append(parsed_event) + + if sync_monitored and settings.general.debug: + trace(f"Skipped {skipped_count} unmonitored events out of {len(events)} for league {league_id}") + + # Remove old events from DB + events_to_delete = list(set(current_events_id_db_list) - set(current_events_sportarr)) + + for event_key in events_to_delete: + try: + database.execute( + delete(TableSportsEvents) + .where(TableSportsEvents.sportarrEventId == event_key[0]) + .where(TableSportsEvents.partNumber == event_key[1])) + except IntegrityError as e: + logging.error(f"BAZARR cannot delete events because of {e}") + else: + event_stream(type='event', action='delete', payload=event_key[0]) + + # Insert new events in DB + for added_event in events_to_add: + try: + added_event['created_at_timestamp'] = datetime.now() + database.execute(insert(TableSportsEvents).values(added_event)) + except IntegrityError as e: + logging.error(f"BAZARR cannot insert events because of {e}. We'll try to update it instead.") + del added_event['created_at_timestamp'] + events_to_update.append(added_event) + else: + row_id = get_event_row_id(added_event['sportarrEventId'], added_event['partNumber']) + if row_id: + store_subtitles_sports(row_id) + event_stream(type='event', payload=added_event['sportarrEventId']) + + # Update existing events in DB + for updated_event in events_to_update: + try: + previous_event_data = database.execute( + select(TableSportsEvents.id, TableSportsEvents.file_id, TableSportsEvents.path) + .where(TableSportsEvents.sportarrEventId == updated_event['sportarrEventId']) + .where(TableSportsEvents.partNumber == updated_event['partNumber']) + ).first() + + updated_event['updated_at_timestamp'] = datetime.now() + database.execute( + update(TableSportsEvents) + .values(updated_event) + .where(TableSportsEvents.sportarrEventId == updated_event['sportarrEventId']) + .where(TableSportsEvents.partNumber == updated_event['partNumber'])) + except IntegrityError as e: + logging.error(f"BAZARR cannot update events because of {e}") + else: + if previous_event_data and (previous_event_data.file_id != updated_event['file_id'] or + previous_event_data.path != updated_event['path']): + # Sportarr gives a new file_id when it replaces a file. The path can + # stay the same through an upgrade, so the id is what proves the + # media changed. + logging.debug(f'BAZARR updating subtitles for event ' + f'{path_mappings.path_replace_sports(updated_event["path"])}') + store_subtitles_sports(previous_event_data.id) + event_stream(type='event', action='update', payload=updated_event['sportarrEventId']) + + league_title = database.execute( + select(TableSportsLeagues.title) + .where(TableSportsLeagues.sportarrLeagueId == league_id)).first() + if league_title: + logging.debug(f'BAZARR All events synced from Sportarr into database for {league_title[0]}.') diff --git a/bazarr/sportarr/sync/leagues.py b/bazarr/sportarr/sync/leagues.py index acb5d17d10..8825cd212c 100644 --- a/bazarr/sportarr/sync/leagues.py +++ b/bazarr/sportarr/sync/leagues.py @@ -13,6 +13,7 @@ from app.event_handler import event_stream from app.jobs_queue import jobs_queue +from .events import sync_events from .parser import leagueParser from .utils import get_tags, get_leagues_from_sportarr_api @@ -120,6 +121,9 @@ def update_leagues(job_id=None, wait_for_completion=False): update_one_league(league['id'], action='updated', league_data=[league], tagsDict=tagsDict, language_profiles=language_profiles) + # Update events in DB + sync_events(league_id=league['id']) + # Calculate leagues to remove from DB removed_leagues = list(set(current_leagues_db) - set(current_leagues_sportarr)) diff --git a/bazarr/sportarr/sync/parser.py b/bazarr/sportarr/sync/parser.py index 6c41a48f80..d59f62a8f1 100644 --- a/bazarr/sportarr/sync/parser.py +++ b/bazarr/sportarr/sync/parser.py @@ -4,6 +4,54 @@ from utilities.path_mappings import path_mappings +def eventParser(event, file): + """Build one row from one playable file. + + A card ships prelims and a main card as separate files under a single + event, and each needs its own subtitles, so the file is the unit here and + the event is what groups them. + """ + video_format = None + video_resolution = None + if file.get('quality'): + # Sportarr reports quality as source and resolution joined, for + # example WEBDL-1080p, which is the shape Sonarr uses too. + parts = file['quality'].split('-') + video_format = parts[0] + if len(parts) > 1: + video_resolution = parts[1] + + return { + 'sportarrLeagueId': event['leagueId'], + 'sportarrEventId': event['id'], + # Stable event id from Sportarr, unlike the integer above which is + # local to one install. + 'externalId': event.get('externalId'), + 'title': event['title'], + 'path': file['filePath'], + 'season': event['seasonNumber'], + 'episode': event['episodeNumber'], + 'broadcastDate': event.get('broadcastDate') or event.get('eventDate'), + 'partName': file.get('partName'), + # 0 keeps a single-file event distinct from a part, so the key stays + # unique either way. + 'partNumber': file.get('partNumber') or 0, + # Set only when a real grab produced the file. Null means there is no + # release to match, so a consumer falls back to the file hash. + 'sceneName': file.get('releaseTitle'), + 'monitored': str(bool(event['monitored'])), + 'format': video_format, + 'resolution': video_resolution, + 'video_codec': file.get('codec'), + 'audio_codec': file.get('audioCodec'), + # Changes when a quality upgrade replaces the file, which is how a + # consumer knows the media changed while the path stayed the same. + 'file_id': file['id'], + 'audio_language': str(file.get('languages') or []), + 'file_size': file.get('size'), + } + + def leagueParser(league, action, tags_dict, language_profiles, league_default_profile): overview = league['overview'] if 'overview' in league else '' poster = league.get('posterUrl') or '' diff --git a/bazarr/sportarr/sync/utils.py b/bazarr/sportarr/sync/utils.py index 019ff51b68..704a8330b0 100644 --- a/bazarr/sportarr/sync/utils.py +++ b/bazarr/sportarr/sync/utils.py @@ -31,6 +31,75 @@ def get_tags(): return tagsDict.json() +def get_event_from_sportarr_api(apikey_sportarr, event_id): + """Read a single event, mainly to learn which league it belongs to. + + Some stream frames name only the event, so the league has to be looked up + before anything can be synced. + """ + url_sportarr_api_event = f"{url_api_sportarr()}events/{event_id}?apikey={apikey_sportarr}" + + try: + r = requests.get(url_sportarr_api_event, timeout=int(settings.sportarr.http_timeout), verify=False, + headers=HEADERS) + r.raise_for_status() + except requests.exceptions.RequestException: + logging.exception(f"BAZARR Error trying to get event {event_id} from Sportarr.") + return + except Exception as e: + logging.exception(f"Exception raised while getting event from Sportarr API: {e}") + return + else: + return r.json() + + +def get_events_from_sportarr_api(apikey_sportarr, league_id): + """Walk every page of a league's events. + + A season can run to a few thousand events, so this pages rather than + asking for the lot. Returning None on any failure matters, because a + partial walk read as a complete one looks like the league lost most of + its events, and the caller would delete them. + """ + events = [] + page = 1 + + while True: + url_sportarr_api_events = (f"{url_api_sportarr()}leagues/{league_id}/events?" + f"page={page}&pageSize=1000&apikey={apikey_sportarr}") + + try: + r = requests.get(url_sportarr_api_events, timeout=int(settings.sportarr.http_timeout), verify=False, + headers=HEADERS) + r.raise_for_status() + except requests.exceptions.HTTPError as e: + if e.response.status_code: + logging.exception(f"BAZARR Error trying to get events from Sportarr. HTTP error " + f"{e.response.status_code}") + return + except requests.exceptions.ConnectionError: + logging.exception("BAZARR Error trying to get events from Sportarr. Connection Error.") + return + except requests.exceptions.Timeout: + logging.exception("BAZARR Error trying to get events from Sportarr. Timeout Error.") + return + except requests.exceptions.RequestException: + logging.exception("BAZARR Error trying to get events from Sportarr.") + return + except Exception as e: + logging.exception(f"Exception raised while getting events from Sportarr API: {e}") + return + + result = r.json() + events += result.get('records', []) + + if page >= result.get('totalPages', 1): + break + page += 1 + + return events + + def get_leagues_from_sportarr_api(apikey_sportarr, sportarr_league_id=None): url_sportarr_api_leagues = (f"{url_api_sportarr()}leagues/{sportarr_league_id if sportarr_league_id else ''}?" f"apikey={apikey_sportarr}") diff --git a/bazarr/subtitles/download.py b/bazarr/subtitles/download.py index 34380a96f0..e460e2f47c 100644 --- a/bazarr/subtitles/download.py +++ b/bazarr/subtitles/download.py @@ -12,7 +12,7 @@ from subliminal_patch.core_persistent import download_best_subtitles from app.config import settings, get_array_from -from app.database import TableEpisodes, TableMovies, database, select, get_profiles_list +from app.database import TableEpisodes, TableMovies, TableSportsEvents, database, select, get_profiles_list from utilities.path_mappings import path_mappings from utilities.helper import get_target_folder, force_unicode from languages.get_languages import alpha3_from_alpha2 @@ -178,6 +178,14 @@ def parse_language_object(language): return language +def _reverse_path(path, media_type): + if media_type == 'series': + return path_mappings.path_replace_reverse(path) + elif media_type == 'sports': + return path_mappings.path_replace_reverse_sports(path) + return path_mappings.path_replace_reverse_movie(path) + + def check_missing_languages(path, media_type): # confirm if language is still missing or if cutoff has been reached if media_type == 'series': @@ -185,6 +193,11 @@ def check_missing_languages(path, media_type): select(TableEpisodes.missing_subtitles) .where(TableEpisodes.path == path_mappings.path_replace_reverse(path)))\ .first() + elif media_type == 'sports': + confirmed_missing_subs = database.execute( + select(TableSportsEvents.missing_subtitles) + .where(TableSportsEvents.path == path_mappings.path_replace_reverse_sports(path)))\ + .first() else: confirmed_missing_subs = database.execute( select(TableMovies.missing_subtitles) @@ -192,8 +205,7 @@ def check_missing_languages(path, media_type): .first() if not confirmed_missing_subs: - reversed_path = path_mappings.path_replace_reverse(path) if media_type == 'series' else \ - path_mappings.path_replace_reverse_movie(path) + reversed_path = _reverse_path(path, media_type) logging.debug(f"BAZARR no media with this path have been found in database: {reversed_path}") return [] diff --git a/bazarr/subtitles/indexer/sports.py b/bazarr/subtitles/indexer/sports.py new file mode 100644 index 0000000000..a8b521e468 --- /dev/null +++ b/bazarr/subtitles/indexer/sports.py @@ -0,0 +1,352 @@ +# coding=utf-8 + +import gc +import os +import logging + +from subliminal_patch import core, search_external_subtitles + +from languages.custom_lang import CustomLanguage +from app.database import get_profiles_list, get_profile_cutoff, TableSportsEvents, TableSportsLeagues, \ + TableSportsEventsSubtitles, get_audio_profile_languages, get_sports_subtitles, database, update, select, insert, \ + delete +from languages.get_languages import alpha2_from_alpha3, get_language_set +from app.config import settings +from utilities.helper import get_subtitle_destination_folder +from utilities.path_mappings import path_mappings +from utilities.video_analyzer import embedded_subs_reader +from app.event_handler import event_stream +from subtitles.indexer.utils import guess_external_subtitles, get_external_subtitles_path +from app.jobs_queue import jobs_queue + +gc.enable() + + +def store_subtitles_sports(sports_event_id, use_cache=True): + item = database.execute( + select(TableSportsEvents.sportarrLeagueId, + TableSportsEvents.path, + TableSportsEvents.file_id, + TableSportsEvents.file_size) + .where(TableSportsEvents.id == sports_event_id) + ).first() + + if not item: + logging.warning(f"BAZARR could not find sports event with ID {sports_event_id} in the database.") + return + else: + original_path = item.path + mapped_path = path_mappings.path_replace_sports(original_path) + + logging.debug(f'BAZARR started subtitles indexing for this file: {mapped_path}') + embedded_subtitles = [] + external_subtitles = [] + + if os.path.exists(mapped_path): + if settings.general.use_embedded_subs: + logging.debug("BAZARR is trying to index embedded subtitles.") + try: + subtitle_languages = embedded_subs_reader(mapped_path, + file_size=item.file_size, + sports_event_id=sports_event_id, + use_cache=use_cache) + for track_id, subtitle_language, subtitle_forced, subtitle_hi, subtitle_codec in subtitle_languages: + try: + if (settings.general.ignore_pgs_subs and subtitle_codec.lower() == "pgs") or \ + (settings.general.ignore_vobsub_subs and subtitle_codec.lower() == "vobsub") or \ + (settings.general.ignore_ass_subs and subtitle_codec.lower() == "ass"): + logging.debug(f"BAZARR skipping {subtitle_codec} sub for language: " + f"{alpha2_from_alpha3(subtitle_language)}") + continue + + if alpha2_from_alpha3(subtitle_language) is not None: + lang = alpha2_from_alpha3(subtitle_language) + logging.debug(f"BAZARR embedded subtitles detected: {lang}" + f"{':forced' if subtitle_forced else ''}{':hi' if subtitle_hi else ''}") + embedded_subtitles.append({'sportarrLeagueId': item.sportarrLeagueId, + 'sportsEventId': sports_event_id, + 'language': lang, + 'forced': subtitle_forced, + 'hi': subtitle_hi, + 'embedded_track_id': track_id}) + except Exception as error: + logging.debug(f"BAZARR unable to index this unrecognized language: {subtitle_language} " + f"({error})") + + database.execute( + delete(TableSportsEventsSubtitles) + .where(TableSportsEventsSubtitles.sportsEventId == sports_event_id) + .where(TableSportsEventsSubtitles.path.is_(None)) + .where(TableSportsEventsSubtitles.embedded_track_id.is_(None)) + ) + + embedded_subtitles_id_list = [] + + if len(embedded_subtitles): + embedded_stmt = insert(TableSportsEventsSubtitles).values(embedded_subtitles) + embedded_stmt = embedded_stmt.on_conflict_do_update( + index_elements=['embedded_track_id', 'sportarrLeagueId', 'sportsEventId', 'language', + 'forced', 'hi'], + set_={ + 'language': embedded_stmt.excluded.language, + 'forced': embedded_stmt.excluded.forced, + 'hi': embedded_stmt.excluded.hi, + 'size': embedded_stmt.excluded.size, + 'embedded_track_id': embedded_stmt.excluded.embedded_track_id + }, + index_where=TableSportsEventsSubtitles.path.is_(None) + ) + database.execute(embedded_stmt) + embedded_subtitles_id_list = [x['embedded_track_id'] for x in embedded_subtitles] + + database.execute( + delete(TableSportsEventsSubtitles) + .where(TableSportsEventsSubtitles.sportsEventId == sports_event_id) + .where(TableSportsEventsSubtitles.path.is_(None)) + .where(TableSportsEventsSubtitles.embedded_track_id.not_in(embedded_subtitles_id_list)) + ) + except Exception: + logging.exception(f"BAZARR error when trying to analyze this {os.path.splitext(mapped_path)[1]} file: " + f"{mapped_path}") + + try: + dest_folder = get_subtitle_destination_folder() + core.CUSTOM_PATHS = [dest_folder] if dest_folder else [] + + previously_indexed_subtitles = get_sports_subtitles(sports_event_id=sports_event_id) + + previously_indexed_subtitles_to_delete = \ + [path_mappings.path_replace_reverse_sports(x['path']) for x in previously_indexed_subtitles + if x['path'] and not os.path.isfile(x['path'])] + + if previously_indexed_subtitles_to_delete: + database.execute( + delete(TableSportsEventsSubtitles) + .where(TableSportsEventsSubtitles.path.in_(previously_indexed_subtitles_to_delete))) + + subtitles = search_external_subtitles(mapped_path, languages=get_language_set(), + only_one=settings.general.single_language) + full_dest_folder_path = os.path.dirname(mapped_path) + if dest_folder: + if settings.general.subfolder == "absolute": + full_dest_folder_path = dest_folder + elif settings.general.subfolder == "relative": + full_dest_folder_path = os.path.join(os.path.dirname(mapped_path), dest_folder) + subtitles = guess_external_subtitles(full_dest_folder_path, subtitles, + previously_indexed_subtitles_to_exclude=previously_indexed_subtitles) + except Exception as e: + logging.exception(f"BAZARR unable to index external subtitles for this file {mapped_path}: {repr(e)}") + else: + for subtitle, language in subtitles.items(): + subtitle_path = get_external_subtitles_path(mapped_path, subtitle) + + try: + subtitle_size = os.stat(subtitle_path).st_size + except FileNotFoundError: + logging.debug(f"BAZARR skipping missing subtitle file: {subtitle_path}") + continue + + custom = CustomLanguage.found_external(subtitle, subtitle_path) + if custom is not None: + logging.debug(f"BAZARR external subtitles detected: {custom}") + external_subtitles.append({'sportarrLeagueId': item.sportarrLeagueId, + 'sportsEventId': sports_event_id, + 'language': custom.split(':')[0], + 'forced': custom.endswith(':forced'), + 'hi': custom.endswith(':hi'), + 'path': path_mappings.path_replace_reverse_sports(subtitle_path), + 'size': subtitle_size}) + + elif str(language.basename) != 'und': + logging.debug(f"BAZARR external subtitles detected: {language}" + f"{':forced' if language.forced else ''}" + f"{':hi' if language.hi else ''}") + external_subtitles.append({'sportarrLeagueId': item.sportarrLeagueId, + 'sportsEventId': sports_event_id, + 'language': language.basename, + 'forced': language.forced, + 'hi': language.hi, + 'path': path_mappings.path_replace_reverse_sports(subtitle_path), + 'size': subtitle_size}) + + if len(external_subtitles): + stmt = insert(TableSportsEventsSubtitles).values(external_subtitles) + stmt = stmt.on_conflict_do_update( + index_elements=['path', 'sportarrLeagueId', 'sportsEventId', 'language', 'forced', 'hi'], + set_={ + 'language': stmt.excluded.language, + 'forced': stmt.excluded.forced, + 'hi': stmt.excluded.hi, + 'size': stmt.excluded.size + } + ) + database.execute(stmt) + else: + logging.debug("BAZARR this file doesn't seems to exist or isn't accessible.") + return + + logging.debug(f"BAZARR has stored those languages to DB: {embedded_subtitles + external_subtitles}") + + list_missing_subtitles_sports(evno=sports_event_id) + + logging.debug(f'BAZARR ended subtitles indexing for this file: {mapped_path}') + + +def list_missing_subtitles_sports(no=None, evno=None): + stmt = select(TableSportsLeagues.sportarrLeagueId, + TableSportsEvents.id.label("sportsEventId"), + TableSportsLeagues.profileId, + TableSportsEvents.audio_language) \ + .select_from(TableSportsEvents) \ + .join(TableSportsLeagues) + + if evno is not None: + events_subtitles = database.execute(stmt.where(TableSportsEvents.id == evno)).all() + elif no is not None: + events_subtitles = database.execute(stmt.where(TableSportsEvents.sportarrLeagueId == no)).all() + else: + events_subtitles = database.execute(stmt).all() + + use_embedded_subs = settings.general.use_embedded_subs + + for event_subtitles in events_subtitles: + def matches_audio(language): + return any(x['code2'] == language['language'] + for x in get_audio_profile_languages(event_subtitles.audio_language)) + + missing_subtitles_text = '[]' + if event_subtitles.profileId: + # get desired subtitles + desired_subtitles_temp = get_profiles_list(profile_id=event_subtitles.profileId) + desired_subtitles_list = [] + if desired_subtitles_temp: + for language in desired_subtitles_temp['items']: + if language['audio_exclude'] == "True": + if matches_audio(language): + continue + if language['audio_only_include'] == "True": + if not matches_audio(language): + continue + desired_subtitles_list.append({'language': language['language'], + 'forced': str(language['forced']), + 'hi': str(language['hi'])}) + + # get existing subtitles + actual_subtitles_list = [] + actual_subtitles_temp = get_sports_subtitles(sports_event_id=event_subtitles.sportsEventId) + if not use_embedded_subs: + actual_subtitles_temp = [x for x in actual_subtitles_temp if x['path']] + + for subtitles in actual_subtitles_temp: + actual_subtitles_list.append({'language': subtitles['code2'], + 'forced': str(subtitles['forced']), + 'hi': str(subtitles['hi'])}) + + # check if cutoff is reached and skip any further check + cutoff_met = False + cutoff_temp_list = get_profile_cutoff(profile_id=event_subtitles.profileId) + + if cutoff_temp_list: + for cutoff_temp in cutoff_temp_list: + cutoff_language = {'language': cutoff_temp['language'], + 'forced': cutoff_temp['forced'], + 'hi': cutoff_temp['hi']} + if cutoff_temp['audio_only_include'] == 'True' and not matches_audio(cutoff_temp): + # We don't want subs in this language unless it matches + # the audio. Don't use it to meet the cutoff. + continue + elif cutoff_temp['audio_exclude'] == 'True' and matches_audio(cutoff_temp): + # The cutoff is met through one of the audio tracks. + cutoff_met = True + elif cutoff_language in actual_subtitles_list: + cutoff_met = True + # HI is considered as good as normal + elif (cutoff_language and + {'language': cutoff_language['language'], + 'forced': 'False', + 'hi': 'True'} in actual_subtitles_list): + cutoff_met = True + + if cutoff_met: + missing_subtitles_text = str([]) + else: + # if cutoff isn't met or None, we continue + + # get difference between desired and existing subtitles + missing_subtitles_list = [] + for item in desired_subtitles_list: + if item not in actual_subtitles_list: + missing_subtitles_list.append(item) + + # remove missing that have hi subtitles for this language in existing + for item in actual_subtitles_list: + if item['hi'] == 'True': + try: + missing_subtitles_list.remove({'language': item['language'], + 'forced': 'False', + 'hi': 'False'}) + except ValueError: + pass + + # make the missing languages list looks like expected + missing_subtitles_output_list = [] + for item in missing_subtitles_list: + lang = item['language'] + if item['forced'] == 'True': + lang += ':forced' + elif item['hi'] == 'True': + lang += ':hi' + missing_subtitles_output_list.append(lang) + + missing_subtitles_text = str(missing_subtitles_output_list) + + database.execute( + update(TableSportsEvents) + .values(missing_subtitles=missing_subtitles_text) + .where(TableSportsEvents.id == event_subtitles.sportsEventId)) + + event_stream(type='sports-event', payload=event_subtitles.sportsEventId) + event_stream(type='sports-event-wanted', action='update', payload=event_subtitles.sportsEventId) + event_stream(type='badges') + + +def sports_full_scan_subtitles(job_id=None, use_cache=None, wait_for_completion=False): + if not job_id: + jobs_queue.add_job_from_function("Indexing all existing sports events subtitles", is_progress=True, + wait_for_completion=wait_for_completion) + return + + if use_cache is None: + use_cache = settings.sportarr.use_ffprobe_cache + + events = database.execute( + select(TableSportsEvents.path, + TableSportsLeagues.title, + TableSportsEvents.title.label("eventTitle"), + TableSportsEvents.id) + .select_from(TableSportsEvents) + .join(TableSportsLeagues) + ).all() + + jobs_queue.update_job_progress(job_id=job_id, progress_max=len(events), progress_message='Indexing') + for i, event in enumerate(events, start=1): + jobs_queue.update_job_progress(job_id=job_id, progress_value=i, + progress_message=f"{event.title} - {event.eventTitle}") + store_subtitles_sports(event.id, use_cache=use_cache) + + logging.info('BAZARR All existing sports events subtitles indexed from disk.') + + jobs_queue.update_job_name(job_id=job_id, new_job_name="Indexed all existing sports events subtitles") + + gc.collect() + + +def sports_scan_subtitles(no): + events = database.execute( + select(TableSportsEvents.id) + .where(TableSportsEvents.sportarrLeagueId == no) + .order_by(TableSportsEvents.id)) \ + .all() + + for event in events: + store_subtitles_sports(event.id, use_cache=False) diff --git a/bazarr/subtitles/mass_download/__init__.py b/bazarr/subtitles/mass_download/__init__.py index 7fe746a2b7..6a7d3361a2 100644 --- a/bazarr/subtitles/mass_download/__init__.py +++ b/bazarr/subtitles/mass_download/__init__.py @@ -2,3 +2,4 @@ from .movies import movies_download_subtitles # noqa: W0611 from .series import series_download_subtitles, episode_download_subtitles # noqa: W0611 +from .sports import league_download_subtitles, sports_event_download_subtitles # noqa: W0611 diff --git a/bazarr/subtitles/mass_download/sports.py b/bazarr/subtitles/mass_download/sports.py new file mode 100644 index 0000000000..4d2b26a2f7 --- /dev/null +++ b/bazarr/subtitles/mass_download/sports.py @@ -0,0 +1,186 @@ +# coding=utf-8 +# fmt: off + +import ast +import logging +import operator +import os + +from functools import reduce + +from utilities.path_mappings import path_mappings +from subtitles.indexer.sports import store_subtitles_sports, list_missing_subtitles_sports +from sportarr.history import history_log_sports +from app.get_providers import get_providers +from app.database import (get_exclusion_clause, get_audio_profile_languages, TableSportsLeagues, TableSportsEvents, + database, select, get_sports_subtitles) +from app.jobs_queue import jobs_queue +from app.config import settings + +from ..download import generate_subtitles + + +def _event_label(event): + # The part identifies which file of the event this is. Most events have one + # file and no part name. + part = f' - {event.partName}' if event.partName else '' + return f'{event.title} - {event.eventTitle}{part}' + + +def league_download_subtitles(no, job_id=None, job_sub_function=False): + if not job_sub_function and not job_id: + jobs_queue.add_job_from_function(f"""Downloading missing subtitles for {database.scalar( + select(TableSportsLeagues.title).where(TableSportsLeagues.sportarrLeagueId == no)) + or 'Unknown League'}""", is_progress=True) + return + + league_row = database.execute( + select(TableSportsLeagues.path, + TableSportsLeagues.title) + .where(TableSportsLeagues.sportarrLeagueId == no))\ + .first() + + if league_row and not os.path.exists(path_mappings.path_replace_sports(league_row.path)): + raise OSError + + conditions = [(TableSportsEvents.sportarrLeagueId == no), + (TableSportsEvents.missing_subtitles != '[]')] + conditions += get_exclusion_clause('sports') + events_details = database.execute( + select(TableSportsEvents.id, + TableSportsLeagues.title, + TableSportsEvents.partName, + TableSportsEvents.title.label('eventTitle'), + TableSportsEvents.missing_subtitles) + .select_from(TableSportsEvents) + .join(TableSportsLeagues) + .where(reduce(operator.and_, conditions))) \ + .all() + throttled = False + if not events_details: + logging.debug(f"BAZARR no sports event for that league have been found in database or they have all been " + f"ignored because of monitored status, sport or league tags: {no}") + else: + count_events_details = len(events_details) + + jobs_queue.update_job_progress(job_id=job_id, progress_max=count_events_details) + for i, event in enumerate(events_details, start=1): + jobs_queue.update_job_progress(job_id=job_id, progress_value=i, progress_message=_event_label(event)) + + providers_list = get_providers() + fallback_allowed = settings.general.use_whisper_fallback and settings.general.use_whisper_fallback_series + if providers_list: + sports_event_download_subtitles(no=event.id, job_id=job_id, job_sub_function=True, + providers_list=providers_list, fallback_allowed=fallback_allowed) + else: + jobs_queue.update_job_progress(job_id=job_id, progress_value=count_events_details) + logging.info("BAZARR All providers are throttled") + throttled = True + break + + outcome_msg = ("All providers throttled" if throttled + else "Search completed") + jobs_queue.update_job_progress(job_id=job_id, progress_message=outcome_msg) + jobs_queue.update_job_name(job_id=job_id, new_job_name=f"Downloaded missing subtitles for {league_row.title}") + + +def sports_event_download_subtitles(no, job_id=None, job_sub_function=False, providers_list=None, + fallback_allowed=False): + if not job_sub_function and not job_id: + jobs_queue.add_job_from_function(f"""Downloading missing subtitles for {database.scalar( + select(TableSportsEvents.title).where(TableSportsEvents.id == no)) or 'Unknown Event'}""", + is_progress=True) + return + + conditions = [(TableSportsEvents.id == no)] + conditions += get_exclusion_clause('sports') + stmt = select(TableSportsEvents.path, + TableSportsEvents.missing_subtitles, + TableSportsEvents.monitored, + TableSportsEvents.id, + TableSportsEvents.sceneName, + TableSportsLeagues.tags, + TableSportsLeagues.title, + TableSportsLeagues.sportarrLeagueId, + TableSportsEvents.audio_language, + TableSportsLeagues.sport, + TableSportsEvents.title.label('eventTitle'), + TableSportsEvents.partName, + TableSportsLeagues.profileId) \ + .select_from(TableSportsEvents) \ + .join(TableSportsLeagues) \ + .where(reduce(operator.and_, conditions)) + event = database.execute(stmt).first() + + if not event: + logging.debug(f"BAZARR no sports event with that id can be found in database: {no}") + jobs_queue.update_job_progress(job_id=job_id, progress_message="Sports event not found in database.") + return + + previously_indexed_subtitles = get_sports_subtitles(sports_event_id=event.id) + + if not len(previously_indexed_subtitles) or \ + any([not x['embedded_track_id'] for x in previously_indexed_subtitles if not x['path']]): + # subtitles indexing for this event might be incomplete, we'll do it again + store_subtitles_sports(event.id) + event = database.execute(stmt).first() + elif event.missing_subtitles is None: + # missing subtitles calculation for this event is incomplete, we'll do it again + list_missing_subtitles_sports(evno=no) + event = database.execute(stmt).first() + + eventPath = path_mappings.path_replace_sports(event.path) + + if not os.path.exists(eventPath): + logging.debug(f"BAZARR sports event file not found. Path mapping issue?: {eventPath}") + jobs_queue.update_job_progress(job_id=job_id, + progress_message=f"Sports event path doesn't exists: {eventPath}") + raise OSError + + if not providers_list: + providers_list = get_providers() + + downloaded_count = 0 + if providers_list: + audio_language_list = get_audio_profile_languages(event.audio_language) + if len(audio_language_list) > 0: + audio_language = audio_language_list[0]['name'] + else: + audio_language = 'None' + + languages = [] + + if not job_sub_function and job_id: + jobs_queue.update_job_progress(job_id=job_id, progress_max=1, progress_message=_event_label(event)) + + for language in ast.literal_eval(event.missing_subtitles): + if language is not None: + hi_ = "True" if language.endswith(':hi') else "False" + forced_ = "True" if language.endswith(':forced') else "False" + languages.append((language.split(":")[0], hi_, forced_)) + + if languages: + for result in generate_subtitles(eventPath, + languages, + audio_language, + str(event.sceneName), + event.title, + 'sports', + event.profileId, + check_if_still_required=True, + job_id=job_id, + fallback_allowed=fallback_allowed): + if result: + store_subtitles_sports(event.id) + history_log_sports(1, event.sportarrLeagueId, event.id, result) + downloaded_count += 1 + outcome_msg = (f"{downloaded_count} subtitle(s) downloaded" + if downloaded_count else "No subtitles found") + else: + logging.info("BAZARR All providers are throttled") + outcome_msg = "All providers throttled" + + if not job_sub_function and job_id: + jobs_queue.update_job_progress(job_id=job_id, progress_value='max', + progress_message=outcome_msg) + jobs_queue.update_job_name(job_id=job_id, new_job_name=f"Downloaded missing subtitles for {event.title}") diff --git a/bazarr/subtitles/pool.py b/bazarr/subtitles/pool.py index 8742e3c4cf..7a7234f131 100644 --- a/bazarr/subtitles/pool.py +++ b/bazarr/subtitles/pool.py @@ -7,6 +7,7 @@ from inspect import getfullargspec from radarr.blacklist import get_blacklist_movie +from sportarr.blacklist import get_blacklist_sports from sonarr.blacklist import get_blacklist from app.get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool, get_language_equals @@ -14,12 +15,20 @@ # fmt: on +def _blacklist_for(media_type): + if media_type == "series": + return get_blacklist() + elif media_type == "sports": + return get_blacklist_sports() + return get_blacklist_movie() + + def _init_pool(media_type, profile_id=None, providers=None): pool = provider_pool() return pool( providers=providers or get_providers(), provider_configs=get_providers_auth(), - blacklist=get_blacklist() if media_type == "series" else get_blacklist_movie(), + blacklist=_blacklist_for(media_type), throttle_callback=provider_throttle, ban_list=get_ban_list(profile_id), language_hook=None, @@ -55,7 +64,7 @@ def _update_pool(media_type, profile_id=None): return pool.update( get_providers(), get_providers_auth(), - get_blacklist() if media_type == "series" else get_blacklist_movie(), + _blacklist_for(media_type), get_ban_list(profile_id), get_language_equals(), ) @@ -65,7 +74,7 @@ def _pool_update(pool, media_type, profile_id=None): return pool.update( get_providers(), get_providers_auth(), - get_blacklist() if media_type == "series" else get_blacklist_movie(), + _blacklist_for(media_type), get_ban_list(profile_id), get_language_equals(), ) diff --git a/bazarr/subtitles/processing.py b/bazarr/subtitles/processing.py index d2bb009fdb..65bf1fd08c 100644 --- a/bazarr/subtitles/processing.py +++ b/bazarr/subtitles/processing.py @@ -8,10 +8,11 @@ from utilities.post_processing import pp_replace, set_chmod from utilities.autopulse_webhook import call_external_webhook from languages.get_languages import alpha2_from_alpha3, alpha2_from_language, alpha3_from_language, language_from_alpha3 -from app.database import TableShows, TableEpisodes, TableMovies, database, select +from app.database import TableShows, TableEpisodes, TableMovies, TableSportsEvents, database, select from utilities.analytics import event_tracker from radarr.notify import notify_radarr from sonarr.notify import notify_sonarr +from sportarr.notify import notify_sportarr from plex.operations import plex_set_movie_added_date_now, plex_set_episode_added_date_now, plex_refresh_item from jellyfin.operations import jellyfin_refresh_item from app.event_handler import event_stream @@ -100,6 +101,25 @@ def process_subtitle(subtitle, media_type, audio_language, path, max_score, is_u sonarr_series_id=episode_metadata.sonarrSeriesId, sonarr_episode_id=episode_metadata.sonarrEpisodeId, job_id=job_id) + elif media_type == 'sports': + sports_metadata = database.execute( + select(TableSportsEvents.id, TableSportsEvents.sportarrLeagueId, TableSportsEvents.season, + TableSportsEvents.episode) + .where(TableSportsEvents.path == path_mappings.path_replace_reverse_sports(path)))\ + .first() + if not sports_metadata: + return + series_id = sports_metadata.sportarrLeagueId + episode_id = sports_metadata.id + + if sync_checker(subtitle) is True: + from .sync import sync_subtitles + sync_subtitles(video_path=path, srt_path=downloaded_path, + forced=subtitle.language.forced, + hi=subtitle.language.hi, + srt_lang=downloaded_language_code2, + percent_score=percent_score, + job_id=job_id) else: movie_metadata = database.execute( select(TableMovies.radarrId, TableMovies.imdbId, TableMovies.tmdbId) @@ -126,7 +146,9 @@ def process_subtitle(subtitle, media_type, audio_language, path, max_score, is_u percent_score, subtitle_id, downloaded_provider, uploader, release_info, series_id, episode_id) - if media_type == 'series': + if media_type in ('series', 'sports'): + # A sports event is scored as an episode, so it uses the episode + # threshold. The movie one is on a different scale. use_pp_threshold = settings.general.use_postprocessing_threshold pp_threshold = int(settings.general.postprocessing_threshold) else: @@ -162,6 +184,14 @@ def process_subtitle(subtitle, media_type, audio_language, path, max_score, is_u season=episode_metadata.season, episode=episode_metadata.episode, tvdb_id=episode_metadata.tvdbId) + elif media_type == 'sports': + reversed_path = path_mappings.path_replace_reverse_sports(path) + reversed_subtitles_path = path_mappings.path_replace_reverse_sports(downloaded_path) + # Sportarr rescans the league folder, which is how it learns the file is + # there. It has no per-event notify endpoint. + notify_sportarr(sports_metadata.sportarrLeagueId) + event_stream(type='sports-event-history') + event_stream(type='sports-event-wanted', action='delete', payload=sports_metadata.id) else: reversed_path = path_mappings.path_replace_reverse_movie(path) reversed_subtitles_path = path_mappings.path_replace_reverse_movie(downloaded_path) diff --git a/bazarr/subtitles/tools/delete.py b/bazarr/subtitles/tools/delete.py index f02bedf24f..fbfe958e28 100644 --- a/bazarr/subtitles/tools/delete.py +++ b/bazarr/subtitles/tools/delete.py @@ -13,17 +13,20 @@ from utilities.autopulse_webhook import call_external_webhook from subtitles.indexer.series import store_subtitles from subtitles.indexer.movies import store_subtitles_movie +from subtitles.indexer.sports import store_subtitles_sports from subtitles.processing import ProcessSubtitlesResult from sonarr.history import history_log from radarr.history import history_log_movie +from sportarr.history import history_log_sports from sonarr.notify import notify_sonarr from radarr.notify import notify_radarr +from sportarr.notify import notify_sportarr from plex.operations import plex_refresh_item from jellyfin.operations import jellyfin_refresh_item def delete_subtitles(media_type, language, forced, hi, media_path, subtitles_path, sonarr_series_id=None, - sonarr_episode_id=None, radarr_id=None): + sonarr_episode_id=None, radarr_id=None, sportarr_league_id=None, sports_event_id=None): if not subtitles_path: logging.error('No subtitles to delete.') return False @@ -49,6 +52,13 @@ def delete_subtitles(media_type, language, forced, hi, media_path, subtitles_pat select(TableEpisodes.season, TableEpisodes.episode, TableShows.imdbId, TableShows.tvdbId) .join(TableShows) .where(TableEpisodes.sonarrEpisodeId == sonarr_episode_id)).first() + elif media_type == 'sports': + pr = path_mappings.path_replace_sports + prr = path_mappings.path_replace_reverse_sports + + # Sports events carry no media server ids, so there is nothing to + # refresh an item with. The rescan below covers Sportarr. + metadata = None else: pr = path_mappings.path_replace_movie prr = path_mappings.path_replace_reverse_movie @@ -67,6 +77,29 @@ def delete_subtitles(media_type, language, forced, hi, media_path, subtitles_pat reversed_subtitles_path=prr(subtitles_path), hearing_impaired=None) + if media_type == 'sports': + try: + os.remove(pr(subtitles_path)) + except OSError: + logging.exception(f'BAZARR cannot delete subtitles file: {subtitles_path}') + store_subtitles_sports(sports_event_id) + return False + else: + store_subtitles_sports(sports_event_id) + history_log_sports(0, sportarr_league_id, sports_event_id, result) + notify_sportarr(sportarr_league_id) + event_stream(type='sports-league', action='update', payload=sportarr_league_id) + event_stream(type='sports-event-wanted', action='update', payload=sports_event_id) + + call_external_webhook( + subtitle_path=subtitles_path, + media_path=media_path, + language=language_log, + media_type=media_type + ) + + return True + if media_type == 'series': try: os.remove(pr(subtitles_path)) @@ -123,5 +156,5 @@ def delete_subtitles(media_type, language, forced, hi, media_path, subtitles_pat language=language_log, media_type=media_type ) - + return True diff --git a/bazarr/subtitles/utils.py b/bazarr/subtitles/utils.py index 12d1bbdca7..d7efdce04e 100644 --- a/bazarr/subtitles/utils.py +++ b/bazarr/subtitles/utils.py @@ -71,7 +71,9 @@ def _get_lang_obj(alpha3): def _get_scores(media_type, min_movie=None, min_ep=None): - series = "series" == media_type + # A sports event carries a season and an episode number from Sportarr, and + # get_video builds it as an episode, so it is scored as one. + series = media_type in ("series", "sports") handler = DEFAULT_SCORES['episode'] if series else DEFAULT_SCORES['movie'] max_score = MAX_SCORES['episode' if series else 'movie'] diff --git a/bazarr/subtitles/wanted/__init__.py b/bazarr/subtitles/wanted/__init__.py index 73ea6380aa..a1343406cd 100644 --- a/bazarr/subtitles/wanted/__init__.py +++ b/bazarr/subtitles/wanted/__init__.py @@ -2,3 +2,4 @@ from .movies import wanted_download_subtitles_movie, wanted_search_missing_subtitles_movies # noqa: W0611 from .series import wanted_download_subtitles, wanted_search_missing_subtitles_series # noqa: W0611 +from .sports import wanted_download_subtitles_sports, wanted_search_missing_subtitles_sports # noqa: W0611 diff --git a/bazarr/subtitles/wanted/sports.py b/bazarr/subtitles/wanted/sports.py new file mode 100644 index 0000000000..a18ba57504 --- /dev/null +++ b/bazarr/subtitles/wanted/sports.py @@ -0,0 +1,167 @@ +# coding=utf-8 +# fmt: off + +import ast +import logging +import operator +import gc + +from functools import reduce + +from utilities.path_mappings import path_mappings +from subtitles.indexer.sports import store_subtitles_sports, list_missing_subtitles_sports +from sportarr.history import history_log_sports +from app.get_providers import get_providers +from app.database import get_exclusion_clause, get_audio_profile_languages, TableSportsLeagues, TableSportsEvents, \ + database, update, select, get_sports_subtitles +from app.event_handler import event_stream +from app.jobs_queue import jobs_queue +from app.config import settings + +from ..adaptive_searching import is_search_active, updateFailedAttempts +from ..download import generate_subtitles + + +def _wanted_event(event, providers_list, job_id=None): + audio_language_list = get_audio_profile_languages(event.audio_language) + if len(audio_language_list) > 0: + audio_language = audio_language_list[0]['name'] + else: + audio_language = 'None' + + languages = [] + languages_to_stamp = [] + for language in ast.literal_eval(event.missing_subtitles): + if is_search_active(desired_language=language, attempt_string=event.failedAttempts): + hi_ = "True" if language.endswith(':hi') else "False" + forced_ = "True" if language.endswith(':forced') else "False" + languages.append((language.split(":")[0], hi_, forced_)) + languages_to_stamp.append(language) + + else: + logging.debug( + f"BAZARR Search is throttled by adaptive search for this sports event {event.path} and " + f"language: {language}") + + found_any = False + for result in generate_subtitles(path_mappings.path_replace_sports(event.path), + languages, + audio_language, + str(event.sceneName), + event.title, + 'sports', + event.profileId, + check_if_still_required=True, + job_id=job_id, + fallback_allowed=settings.general.use_whisper_fallback): + if result: + found_any = True + store_subtitles_sports(event.id) + history_log_sports(1, event.sportarrLeagueId, event.id, result) + event_stream(type='sports-league', action='update', payload=event.sportarrLeagueId) + event_stream(type='sports-event-wanted', action='delete', payload=event.id) + + if not found_any and providers_list: + for language in languages_to_stamp: + updated = updateFailedAttempts( + desired_language=language, + attempt_string=event.failedAttempts) + database.execute( + update(TableSportsEvents) + .values(failedAttempts=updated) + .where(TableSportsEvents.id == event.id)) + + +def wanted_download_subtitles_sports(sports_event_id, job_id=None): + stmt = select(TableSportsEvents.path, + TableSportsEvents.missing_subtitles, + TableSportsEvents.id, + TableSportsEvents.sportarrLeagueId, + TableSportsEvents.audio_language, + TableSportsEvents.sceneName, + TableSportsEvents.failedAttempts, + TableSportsEvents.title, + TableSportsLeagues.profileId) \ + .select_from(TableSportsEvents) \ + .join(TableSportsLeagues) \ + .where((TableSportsEvents.id == sports_event_id)) + event_details = database.execute(stmt).first() + + previously_indexed_subtitles = get_sports_subtitles(sports_event_id=sports_event_id) + + if not event_details: + logging.debug(f"BAZARR no sports event with that id can be found in database: {sports_event_id}") + return + elif not len(previously_indexed_subtitles) or \ + any([not x['embedded_track_id'] for x in previously_indexed_subtitles if not x['path']]): + # subtitles indexing for this event might be incomplete, we'll do it again + store_subtitles_sports(sports_event_id) + event_details = database.execute(stmt).first() + elif event_details.missing_subtitles is None: + # missing subtitles calculation for this event is incomplete, we'll do it again + list_missing_subtitles_sports(evno=sports_event_id) + event_details = database.execute(stmt).first() + + providers_list = get_providers() + + if providers_list: + _wanted_event(event_details, providers_list, job_id=job_id) + else: + logging.info("BAZARR All providers are throttled") + + +def wanted_search_missing_subtitles_sports(job_id=None, wait_for_completion=False): + if not job_id: + jobs_queue.add_job_from_function("Searching for missing sports events subtitles", is_progress=True, + wait_for_completion=wait_for_completion) + return + + conditions = [(TableSportsEvents.missing_subtitles.is_not(None)), + (TableSportsEvents.missing_subtitles != '[]')] + conditions += get_exclusion_clause('sports') + events = database.execute( + select(TableSportsEvents.sportarrLeagueId, + TableSportsEvents.id, + TableSportsLeagues.tags, + TableSportsEvents.monitored, + TableSportsLeagues.title, + TableSportsEvents.partName, + TableSportsEvents.title.label('eventTitle'), + TableSportsLeagues.sport) + .select_from(TableSportsEvents) + .join(TableSportsLeagues) + .where(reduce(operator.and_, conditions))) \ + .all() + + count_events = len(events) + jobs_queue.update_job_progress(job_id=job_id, progress_max=count_events) + + if count_events == 0: + jobs_queue.update_job_progress(job_id=job_id, progress_value='max') + + throttled = False + for i, event in enumerate(events, start=1): + # The part identifies which file of the event this is, so it belongs in + # the progress message. Most events have one file and no part name. + part = f' - {event.partName}' if event.partName else '' + jobs_queue.update_job_progress(job_id=job_id, progress_value=i, + progress_message=f'{event.title} - {event.eventTitle}{part}') + + providers = get_providers() + if providers: + wanted_download_subtitles_sports(event.id, job_id=job_id) + + # make sure to override the progress value updated by the subtitles synchronization + jobs_queue.update_job_progress(job_id=job_id, progress_value=i, progress_max=count_events) + else: + logging.info("BAZARR All providers are throttled") + throttled = True + break + + outcome_msg = ("All providers throttled" if throttled + else "Search completed") + jobs_queue.update_job_progress(job_id=job_id, progress_message=outcome_msg) + jobs_queue.update_job_name(job_id=job_id, new_job_name="Searched for missing sports events subtitles") + logging.info('BAZARR Finished searching for missing sports events Subtitles. Check History for more information.') + + gc.collect() diff --git a/bazarr/utilities/video_analyzer.py b/bazarr/utilities/video_analyzer.py index 7150c0fbd8..8f25ee9272 100644 --- a/bazarr/utilities/video_analyzer.py +++ b/bazarr/utilities/video_analyzer.py @@ -4,7 +4,7 @@ import pickle from app.config import settings -from app.database import TableEpisodes, TableMovies, database, update, select, get_subtitles +from app.database import TableEpisodes, TableMovies, TableSportsEvents, database, update, select, get_subtitles from languages.custom_lang import CustomLanguage from languages.get_languages import language_from_alpha3, alpha3_from_alpha2 from utilities.path_mappings import path_mappings @@ -36,8 +36,9 @@ def _handle_alpha3(detected_language: dict): return alpha3 -def embedded_subs_reader(file, file_size, episode_file_id=None, movie_file_id=None, use_cache=True): - data = parse_video_metadata(file, file_size, episode_file_id, movie_file_id, use_cache=use_cache) +def embedded_subs_reader(file, file_size, episode_file_id=None, movie_file_id=None, sports_event_id=None, + use_cache=True): + data = parse_video_metadata(file, file_size, episode_file_id, movie_file_id, sports_event_id, use_cache=use_cache) und_default_language = alpha3_from_alpha2(settings.general.default_und_embedded_subtitles_lang) subtitles_list = [] @@ -227,7 +228,8 @@ def subtitles_sync_references(subtitles_path, sonarr_episode_id=None, radarr_mov return references_dict -def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=None, use_cache=True): +def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=None, sports_event_id=None, + use_cache=True): """ This function return the video file properties as parsed by knowit using ffprobe or mediainfo using the cached value by default. @@ -240,6 +242,8 @@ def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=No @param episode_file_id: episode ID of the video file from Sonarr (or None if it's a movie) @type movie_file_id: int or None @param movie_file_id: movie ID of the video file from Radarr (or None if it's an episode) + @type sports_event_id: int or None + @param sports_event_id: row ID of the sports event part from Sportarr (or None if it's not a sports event) @type use_cache: bool @param use_cache: @@ -247,11 +251,13 @@ def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=No @return: return a dictionary including the video file properties as parsed by ffprobe or mediainfo """ + # Sportarr file IDs and Sonarr episode file IDs are unrelated sequences. Keep + # the sports cache in its own table, or a scan writes over an episode's cache. # Define default data keys value data = { "ffprobe": {}, "mediainfo": {}, - "file_id": episode_file_id or movie_file_id, + "file_id": episode_file_id or movie_file_id or sports_event_id, "file_size": file_size, } @@ -269,6 +275,11 @@ def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=No select(TableMovies.ffprobe_cache) .where(TableMovies.movie_file_id == movie_file_id)) \ .first() + elif sports_event_id: + cache_key = database.execute( + select(TableSportsEvents.ffprobe_cache) + .where(TableSportsEvents.id == sports_event_id)) \ + .first() else: cache_key = None @@ -282,7 +293,8 @@ def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=No else: # Check if file size and file id matches and if so, we return the cached value if available for the # desired parser - if cached_value['file_size'] == file_size and cached_value['file_id'] in [episode_file_id, movie_file_id]: + if cached_value['file_size'] == file_size and cached_value['file_id'] in [episode_file_id, movie_file_id, + sports_event_id]: if embedded_subs_parser in cached_value and cached_value[embedded_subs_parser]: return cached_value else: @@ -348,4 +360,9 @@ def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=No update(TableMovies) .values(ffprobe_cache=pickle.dumps(data, pickle.HIGHEST_PROTOCOL)) .where(TableMovies.movie_file_id == movie_file_id)) + elif sports_event_id: + database.execute( + update(TableSportsEvents) + .values(ffprobe_cache=pickle.dumps(data, pickle.HIGHEST_PROTOCOL)) + .where(TableSportsEvents.id == sports_event_id)) return data diff --git a/migrations/versions/c7d1e5a90b32_.py b/migrations/versions/c7d1e5a90b32_.py new file mode 100644 index 0000000000..ad4149f402 --- /dev/null +++ b/migrations/versions/c7d1e5a90b32_.py @@ -0,0 +1,69 @@ +"""empty message + +Revision ID: c7d1e5a90b32 +Revises: a3f2c81b9d47 +Create Date: 2026-08-13 00:41:12.503118 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c7d1e5a90b32' +down_revision = 'a3f2c81b9d47' +branch_labels = None +depends_on = None + + +bind = op.get_context().bind +insp = sa.inspect(bind) +tables = insp.get_table_names() + + +def upgrade(): + if 'table_blacklist_sports' not in tables: + op.create_table( + 'table_blacklist_sports', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('language', sa.Text(), nullable=True), + sa.Column('provider', sa.Text(), nullable=True), + sa.Column('sports_event_id', sa.Integer(), nullable=True), + sa.Column('sportarr_league_id', sa.Integer(), nullable=True), + sa.Column('subs_id', sa.Text(), nullable=True), + sa.Column('timestamp', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['sports_event_id'], ['table_sports_events.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['sportarr_league_id'], ['table_sports_leagues.sportarrLeagueId'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + + if 'table_history_sports' not in tables: + op.create_table( + 'table_history_sports', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('action', sa.Integer(), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('language', sa.Text(), nullable=True), + sa.Column('provider', sa.Text(), nullable=True), + sa.Column('score', sa.Integer(), nullable=True), + sa.Column('score_out_of', sa.Integer(), nullable=True), + sa.Column('sportsEventId', sa.Integer(), nullable=True), + sa.Column('sportarrLeagueId', sa.Integer(), nullable=True), + sa.Column('subs_id', sa.Text(), nullable=True), + sa.Column('subtitles_path', sa.Text(), nullable=True), + sa.Column('timestamp', sa.DateTime(), nullable=False), + sa.Column('video_path', sa.Text(), nullable=True), + sa.Column('matched', sa.Text(), nullable=True), + sa.Column('not_matched', sa.Text(), nullable=True), + sa.Column('upgradedFromId', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['sportsEventId'], ['table_sports_events.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['sportarrLeagueId'], ['table_sports_leagues.sportarrLeagueId'], + ondelete='CASCADE'), + sa.ForeignKeyConstraint(['upgradedFromId'], ['table_history_sports.id'], ), + sa.PrimaryKeyConstraint('id'), + ) + + +def downgrade(): + pass