Use External Service for media storage

[tummyslyunopened]

Use External service for media storage, with graceful migration plan

Initially track external media source side by side.

[tummyslyunopened]

find media references in codebase:

[remote@nixos:/mnt/c/users/remote/dev/kneecap-backend]$ grep -r EDIA --exclude-dir .git --exclude-dir .venv --exclude-dir logs --exclude-dir media
docker-compose.yml:      MEDIA_ROOT: $MEDIA_ROOT
examples/docker-compose.persistent.yml:      MEDIA_ROOT: $MEDIA_ROOT
examples/example.env:MEDIA_ROOT=/path/to/your/media/directory
feeds/generate.py:        rss_file_path = os.path.join(settings.MEDIA_ROOT, f"rss/{subscription_id}.xml")
feeds/generate.py:        rss_file_path = os.path.join(settings.MEDIA_ROOT, "feed/combined_feed.xml")
feeds/management/commands/validate_combined_feed.py:            combined_feed_path = os.path.join(settings.MEDIA_ROOT, 'feed/combined_feed.xml')
kneecap/settings.py:MEDIA_ROOT = validate_env("MEDIA_ROOT", default=os.path.join(BASE_DIR, "media"), redact=False)
kneecap/settings.py:MEDIA_URL = "/media/"  # validate_env("MEDIA_URL", default="/media/", redact=False)
kneecap/urls.py:    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
grep: kneecap/__pycache__/settings.cpython-311.pyc: binary file matches
grep: kneecap/__pycache__/settings.cpython-313.pyc: binary file matches
grep: kneecap/__pycache__/urls.cpython-313.pyc: binary file matches
subscriptions/models.py:        media_root = str(settings.MEDIA_ROOT)
subscriptions/models.py:        audio_url_rel = self.audio_url.replace(settings.MEDIA_URL, "").lstrip("/\\")
subscriptions/models.py:        media_root = str(settings.MEDIA_ROOT)
subscriptions/models.py:        rel_path = self.audio_url.replace(settings.MEDIA_URL, "").lstrip("/\\")
subscriptions/models.py:        media_root = str(settings.MEDIA_ROOT)
subscriptions/models.py:        # Convert file path to URL by replacing MEDIA_ROOT with MEDIA_URL
subscriptions/models.py:        rel_path = os.path.relpath(file_path, settings.MEDIA_ROOT)
subscriptions/models.py:        return settings.MEDIA_URL + rel_path.replace(os.sep, "/")
subscriptions/models.py:        rel_path = os.path.relpath(self.derive_audio_file_path, settings.MEDIA_ROOT)
subscriptions/models.py:        return os.path.normpath(os.path.join(settings.MEDIA_ROOT, transcript_rel))
subscriptions/models.py:        rel_path = os.path.relpath(self.derive_transcript_file_path, settings.MEDIA_ROOT)
subscriptions/models.py:        return settings.MEDIA_URL + rel_path.replace(os.sep, "/")
grep: subscriptions/__pycache__/models.cpython-313.pyc: binary file matches
tools/media.py:        os.makedirs(os.path.join(settings.MEDIA_ROOT, media_path), exist_ok=True)
tools/media.py:        full_path = os.path.join(settings.MEDIA_ROOT, file_path)
tools/media.py:            + settings.MEDIA_URL.rstrip("/")
grep: tools/__pycache__/media.cpython-313.pyc: binary file matches
[remote@nixos:/mnt/c/users/remote/dev/kneecap-backend]$ grep -r edia --exclude-dir .git --exclude-dir .venv --exclude-dir logs --exclude-dir media
.dockerignore:media/
.gitignore:media/
dashboard/templates/feed.html: [Source](https://github.com/tummyslyunopened/kneecap-backend/issues/%7B%7Bepisode.media_link%7D%7D)
dashboard/templates/feed.html:
grep: db.sqlite3: binary file matches
docker-compose.yml: - kneecap_media:/kneecap_media/
docker-compose.yml: kneecap_media:
entrypoint-dev.ps1:Start-Process -FilePath "uv" -ArgumentList "run manage.py download_media" -RedirectStandardOutput "./logs/download-media-output.log" -RedirectStandardError "./logs/download-media-error-log" -NoNewWindow
entrypoint-dev.ps1:Get-Content ./logs/runserver-output.log, ./logs/runserver-error.log, ./logs/sync-rss-data-output.log, ./logs/sync-rss-data-error.log, ./logs/download-media-output.log, ./logs/download-media-error-log, ./logs/transcribe-episodes-output.log, ./logs/transcribe-episodes-error-log, ./logs/generate-low-quality-output.log, ./logs/generate-low-quality-error-log, ./logs/detect-ads-output.log, ./logs/detect-ads-error.log, ./logs/record-minutes-output.log, ./logs/record-minutes-error.log -Wait
entrypoint-dev.sh:uv run manage.py download_media > ./logs/download-media-output.log 2> ./logs/download-media-error-log &
entrypoint-dev.sh: ./logs/download-media-output.log ./logs/download-media-error-log
entrypoint.sh:uv run manage.py download_media > ./logs/download-media-output.log 2> ./logs/download-media-error-log &
entrypoint.sh: ./logs/download-media-output.log ./logs/download-media-error-log
examples/docker-compose.persistent.yml: - kneecap_media:/media/
examples/docker-compose.persistent.yml: kneecap_media:
examples/example.env:# Static and media directories
examples/example.env:MEDIA_ROOT=/path/to/your/media/directory
feeds/generate.py: audio_url = episode.audio_url if episode.audio_url else episode.media_link
feeds/generate.py: audio_url = episode.audio_url if episode.audio_url else episode.media_link
feeds/management/commands/validate_combined_feed.py: field for field in ["title", "pub_date", "description", "media_link"]
kneecap/settings.py:MEDIA_ROOT = validate_env("MEDIA_ROOT", default=os.path.join(BASE_DIR, "media"), redact=False)
kneecap/settings.py:MEDIA_URL = "/media/" # validate_env("MEDIA_URL", default="/media/", redact=False)
grep: kneecap/pycache/settings.cpython-311.pyc: binary file matches
grep: kneecap/pycache/settings.cpython-313.pyc: binary file matches
rss/management/commands/download_media.py: help = "Downloads missing media for all RSS subscriptions."
rss/management/commands/download_media.py: logger = logging.getLogger("rss.management.commands.download_media")
rss/management/commands/download_media.py: self.stdout.write("Starting RSS Entry Media Downloader...")
rss/management/commands/download_media.py: msg = f"Downloading media for episode: {episode.title} (ID: {episode.pk})"
rss/management/commands/download_media.py: f"No media URL or failed download for episode: {episode.title} (ID: {episode.pk})"
rss/management/commands/download_media.py: f"Exception while downloading media for episode {episode.pk}: {e}"
rss/management/commands/refresh_images.py: help = "Downloads missing media for all RSS subscriptions."
grep: rss/management/commands/pycache/download_media.cpython-313.pyc: binary file matches
grep: rss/management/commands/pycache/refresh_images.cpython-313.pyc: binary file matches
rss/models.py:from tools.media import download_media_requests
rss/models.py: success, self.image_url = download_media_requests(
rss/models.py: success, self.rss_url = download_media_requests(
rss/models.py: self.link, str(self.uuid), media_path="rss", default_file_ext=""
rss/models.py: "last_media_download", "media_download_interval", HOUR_SECONDS, HOUR_SECONDS
rss/models.py: if not episode.media_link:
rss/models.py: logger.warning(f"No media URL for episode: {episode.title}")
rss/models.py: success, episode.audio_url = download_media_requests(
rss/models.py: episode.media_link, str(episode.uuid), media_path="episodes", default_file_ext=".mp3"
rss/parsers.py: "media_link": entry.enclosures[0].url if entry.enclosures else None,
rss/parsers.py: "media_link": entry.enclosure_href
rss/tests/test_models.py: @patch("rss.models.download_media_requests")
rss/tests/test_models.py: @patch("rss.models.download_media_requests")
rss/tests/test_models.py: @patch("rss.models.download_media_requests")
rss/tests/test_models.py: @patch("rss.models.download_media_requests")
grep: rss/pycache/models.cpython-313.pyc: binary file matches
grep: rss/pycache/parsers.cpython-313.pyc: binary file matches
scripts/overwite-local-docker-volumes.ps1:docker run -d -v kneecap_db:/db/ -v kneecap_static:/static/ -v kneecap_media:/media/ kneecap/server:0.0
scripts/overwite-local-docker-volumes.ps1:# // Clear the static and media directories
scripts/overwite-local-docker-volumes.ps1:docker exec ${containerId} sh -c "rm -rf /media/*"
scripts/overwite-local-docker-volumes.ps1:# // Recreate the static and media directories
scripts/overwite-local-docker-volumes.ps1:docker exec ${containerId} sh -c "mkdir -p /static /media"
scripts/overwite-local-docker-volumes.ps1:Get-ChildItem -Path ../media | ForEach-Object {
scripts/overwite-local-docker-volumes.ps1: docker cp 
.
F
u
l
l
N
a
m
e
{containerId}:/media/
subscriptions/migrations/0001_initial.py: ("media_link", models.URLField(blank=True)),
subscriptions/migrations/0002_subscription_last_media_download_and_more.py: name="last_media_download",
subscriptions/migrations/0002_subscription_last_media_download_and_more.py: name="media_download_interval",
subscriptions/migrations/0003_episode_transcript_url.py: ("subscriptions", "0002_subscription_last_media_download_and_more"),
grep: subscriptions/migrations/pycache/0001_initial.cpython-313.pyc: binary file matches
grep: subscriptions/migrations/pycache/0002_subscription_last_media_download_and_more.cpython-313.pyc: binary file matches
grep: subscriptions/migrations/pycache/0003_episode_transcript_url.cpython-313.pyc: binary file matches
subscriptions/models.py: last_media_download = models.DateTimeField(blank=True, null=True)
subscriptions/models.py: media_download_interval = models.IntegerField(blank=True, null=True)
subscriptions/models.py: if rss_url_rel.lower().startswith("media/"):
subscriptions/models.py: media_root = str(settings.MEDIA_ROOT)
subscriptions/models.py: return os.path.join(media_root, rss_url_rel)
subscriptions/models.py: media_link = models.URLField(max_length=200, blank=True)
subscriptions/models.py: if audio_url_rel.lower().startswith("media/"):
subscriptions/models.py: media_root = str(settings.MEDIA_ROOT)
subscriptions/models.py: return os.path.normpath(os.path.join(media_root, audio_url_rel))
subscriptions/models.py: # Get the relative path from media root
subscriptions/models.py: if rel_path.lower().startswith("media/"):
subscriptions/models.py: media_root = str(settings.MEDIA_ROOT)
subscriptions/models.py: return os.path.normpath(os.path.join(media_root, low_rel_path))
grep: subscriptions/pycache/models.cpython-313.pyc: binary file matches
timeline/tests/test_minute_model.py: # Try to record another minute immediately (should return None as same minute)
tools/media.py:def get_media_paths(link, uuid, media_path="unsorted", default_file_ext=""):
tools/media.py: os.makedirs(os.path.join(settings.MEDIA_ROOT, media_path), exist_ok=True)
tools/media.py: file_path = os.path.join(media_path, filename)
tools/media.py: logger.warn(f"Failed to generate media full_path and url_path: {e}")
tools/media.py:# @global_throttle("download_media_requests", 1, 1, wait_on_throttled=True)
tools/media.py:def download_media_requests(link, uuid, media_path="unsorted", default_file_ext=""):
tools/media.py: full_path, url_path = get_media_paths(link, str(uuid), media_path, default_file_ext)
tools/media.py: logger.warn(f"failed to download media {e}")
tools/media.py: link, uuid, media_path="unsorted", default_file_ext="", quality_code=""
tools/media.py: full_path, url_path = get_media_paths(link, uuid, media_path, default_file_ext)
tools/storage.py:directory_path = "../media" # Relative path for demonstration
grep: tools/pycache/media.cpython-313.pyc: binary file matches

[tummyslyunopened]

TODO

• add STOWAGE_HOST to kneecap.settings

• modify subscriptions.models (Breaking change requiring workaround migration for existing deployments)

  • add stowage_.url fields & legacy._url fields
  • add management command to copy current .*_url fields to the legacy col
  • Push and run management command on all existing servers
  • Change all .*_url fields to be property methods that can logically switch between stowage and legacy

• tools.models consider implementing intermediate table for external urls. that can make it easier to change between cdns and track external connections.

• duplicate utilities in tools.media to also use external stowage service