F(backend): add public media derivatives pipeline
This commit is contained in:
@@ -5,6 +5,7 @@ from typing import Optional, List
|
||||
from datetime import datetime
|
||||
|
||||
from apps.blog.models import Category, Tag, Comment
|
||||
from core.media import PREVIEW_VARIANT, THUMBNAIL_VARIANT, derivative_url
|
||||
|
||||
|
||||
class CategorySchema(ModelSchema):
|
||||
@@ -23,6 +24,8 @@ class AuthorSchema(Schema):
|
||||
first_name: str
|
||||
last_name: str
|
||||
profile_picture: Optional[str] = None
|
||||
profile_picture_thumbnail_url: Optional[str] = None
|
||||
profile_picture_preview_url: Optional[str] = None
|
||||
|
||||
@staticmethod
|
||||
def resolve_profile_picture(obj, context):
|
||||
@@ -31,6 +34,18 @@ class AuthorSchema(Schema):
|
||||
return request.build_absolute_uri(obj.profile_picture.url)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def resolve_profile_picture_thumbnail_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.profile_picture, THUMBNAIL_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_profile_picture_preview_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.profile_picture, PREVIEW_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
class PostListSchema(Schema):
|
||||
id: int
|
||||
title: str
|
||||
@@ -38,6 +53,9 @@ class PostListSchema(Schema):
|
||||
excerpt: str
|
||||
author: AuthorSchema
|
||||
featured_image: Optional[str] = None
|
||||
absolute_featured_image_url: Optional[str] = None
|
||||
absolute_featured_image_thumbnail_url: Optional[str] = None
|
||||
absolute_featured_image_preview_url: Optional[str] = None
|
||||
status: str
|
||||
published_at: Optional[datetime] = None
|
||||
category: Optional[CategorySchema] = None
|
||||
@@ -46,6 +64,25 @@ class PostListSchema(Schema):
|
||||
created_at: datetime
|
||||
reading_time: int
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_featured_image_url(obj, context):
|
||||
request = context["request"]
|
||||
if obj.featured_image and hasattr(obj.featured_image, "url"):
|
||||
return request.build_absolute_uri(obj.featured_image.url)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_featured_image_thumbnail_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.featured_image, THUMBNAIL_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_featured_image_preview_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.featured_image, PREVIEW_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
class PostDetailSchema(PostListSchema):
|
||||
content: str
|
||||
content_html: str
|
||||
|
||||
@@ -5,6 +5,11 @@ from django.utils import timezone
|
||||
|
||||
import markdown
|
||||
|
||||
from core.media import (
|
||||
delete_image_derivatives_by_name,
|
||||
get_image_previous_name,
|
||||
safe_process_public_image,
|
||||
)
|
||||
from core.models import BaseModel
|
||||
|
||||
class Category(BaseModel):
|
||||
@@ -67,6 +72,9 @@ class Post(BaseModel):
|
||||
return self.title
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
previous_image_name = get_image_previous_name(self, "featured_image")
|
||||
current_image_name = self.featured_image.name if self.featured_image else None
|
||||
|
||||
if not self.slug:
|
||||
self.slug = slugify(self.title)
|
||||
|
||||
@@ -84,6 +92,17 @@ class Post(BaseModel):
|
||||
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
if previous_image_name != current_image_name and previous_image_name:
|
||||
delete_image_derivatives_by_name(
|
||||
self.featured_image.storage if self.featured_image else None,
|
||||
previous_image_name,
|
||||
"blog_featured",
|
||||
delete_original=True,
|
||||
)
|
||||
|
||||
if previous_image_name != current_image_name and self.featured_image:
|
||||
safe_process_public_image(self.featured_image, "blog_featured")
|
||||
|
||||
@property
|
||||
def content_html(self):
|
||||
"""Convert markdown content to HTML"""
|
||||
|
||||
@@ -10,6 +10,7 @@ from apps.blog.api.schemas import AuthorSchema
|
||||
from apps.events.models import Event, Registration
|
||||
from apps.gallery.models import Gallery
|
||||
from apps.payments.models import Payment
|
||||
from core.media import BLUR_VARIANT, PREVIEW_VARIANT, THUMBNAIL_VARIANT, derivative_url
|
||||
|
||||
|
||||
class EventGallerySchema(ModelSchema):
|
||||
@@ -18,6 +19,8 @@ class EventGallerySchema(ModelSchema):
|
||||
file_size_mb: float
|
||||
markdown_url: str
|
||||
absolute_image_url: Optional[str] = None
|
||||
absolute_image_preview_url: Optional[str] = None
|
||||
absolute_image_blur_url: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
model = Gallery
|
||||
@@ -31,12 +34,26 @@ class EventGallerySchema(ModelSchema):
|
||||
return request.build_absolute_uri(obj.image.url)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_image_preview_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.image, PREVIEW_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_image_blur_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.image, BLUR_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
class EventSchema(ModelSchema):
|
||||
"""Schema providing full event details for API responses."""
|
||||
gallery_images: List[EventGallerySchema]
|
||||
description_html: str
|
||||
registration_count: int
|
||||
absolute_featured_image_url: Optional[str] = None
|
||||
absolute_featured_image_thumbnail_url: Optional[str] = None
|
||||
absolute_featured_image_preview_url: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
model = Event
|
||||
@@ -54,6 +71,18 @@ class EventSchema(ModelSchema):
|
||||
return request.build_absolute_uri(obj.featured_image.url)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_featured_image_thumbnail_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.featured_image, THUMBNAIL_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_featured_image_preview_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.featured_image, PREVIEW_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_registration_count(obj):
|
||||
return obj.registrations.filter(status__in=[Registration.StatusChoices.CONFIRMED, Registration.StatusChoices.ATTENDED]).count()
|
||||
@@ -70,6 +99,8 @@ class EventListSchema(Schema):
|
||||
slug: str
|
||||
featured_image: Optional[str] = None
|
||||
absolute_featured_image_url: Optional[str] = None
|
||||
absolute_featured_image_thumbnail_url: Optional[str] = None
|
||||
absolute_featured_image_preview_url: Optional[str] = None
|
||||
event_type: str
|
||||
start_time: datetime
|
||||
end_time: datetime
|
||||
@@ -88,6 +119,18 @@ class EventListSchema(Schema):
|
||||
return request.build_absolute_uri(obj.featured_image.url)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_featured_image_thumbnail_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.featured_image, THUMBNAIL_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_featured_image_preview_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.featured_image, PREVIEW_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_registration_count(obj):
|
||||
return obj.registrations.filter(status__in=[Registration.StatusChoices.CONFIRMED, Registration.StatusChoices.ATTENDED]).count()
|
||||
|
||||
@@ -10,6 +10,11 @@ import uuid
|
||||
import markdown
|
||||
from location_field.models.plain import PlainLocationField as LocationField
|
||||
|
||||
from core.media import (
|
||||
delete_image_derivatives_by_name,
|
||||
get_image_previous_name,
|
||||
safe_process_public_image,
|
||||
)
|
||||
from core.models import BaseModel
|
||||
|
||||
|
||||
@@ -68,10 +73,24 @@ class Event(BaseModel):
|
||||
return self.title
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
previous_image_name = get_image_previous_name(self, "featured_image")
|
||||
current_image_name = self.featured_image.name if self.featured_image else None
|
||||
|
||||
if not self.slug:
|
||||
self.slug = slugify(self.title)
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
if previous_image_name != current_image_name and previous_image_name:
|
||||
delete_image_derivatives_by_name(
|
||||
self.featured_image.storage if self.featured_image else None,
|
||||
previous_image_name,
|
||||
"event_featured",
|
||||
delete_original=True,
|
||||
)
|
||||
|
||||
if previous_image_name != current_image_name and self.featured_image:
|
||||
safe_process_public_image(self.featured_image, "event_featured")
|
||||
|
||||
@property
|
||||
def description_html(self):
|
||||
"""Convert markdown description to HTML"""
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Optional
|
||||
|
||||
from apps.blog.api.schemas import AuthorSchema
|
||||
from apps.gallery.models import Gallery
|
||||
from core.media import BLUR_VARIANT, PREVIEW_VARIANT, derivative_url
|
||||
|
||||
|
||||
class GallerySchema(ModelSchema):
|
||||
@@ -12,12 +13,34 @@ class GallerySchema(ModelSchema):
|
||||
uploaded_by: AuthorSchema
|
||||
file_size_mb: float
|
||||
markdown_url: str
|
||||
absolute_image_url: Optional[str] = None
|
||||
absolute_image_preview_url: Optional[str] = None
|
||||
absolute_image_blur_url: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
model = Gallery
|
||||
model_fields = ['id', 'title', 'description', 'image', 'alt_text',
|
||||
'width', 'height', 'is_public', 'created_at']
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_image_url(obj, context):
|
||||
request = context["request"]
|
||||
if obj.image and hasattr(obj.image, "url"):
|
||||
return request.build_absolute_uri(obj.image.url)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_image_preview_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.image, PREVIEW_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_absolute_image_blur_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.image, BLUR_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
|
||||
class GalleryCreateSchema(Schema):
|
||||
"""Payload for creating a gallery entry."""
|
||||
|
||||
@@ -63,6 +63,7 @@ def upload_image(request, file: UploadedFile = File(...), data: GalleryCreateSch
|
||||
alt_text=data.alt_text if data else "",
|
||||
is_public=data.is_public if data else True
|
||||
)
|
||||
gallery_item._defer_image_processing = True
|
||||
|
||||
# Save image
|
||||
filename = f"gallery/{uuid.uuid4().hex}.{file.name.split('.')[-1]}"
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
from django.db import models
|
||||
from django.conf import settings
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from core.models import BaseModel
|
||||
from core.media import (
|
||||
delete_image_derivatives_by_name,
|
||||
get_image_previous_name,
|
||||
safe_process_public_image,
|
||||
)
|
||||
|
||||
|
||||
MAX_IMAGE_FILE_SIZE_BYTES = 2 * 1024 * 1024
|
||||
|
||||
class Gallery(BaseModel):
|
||||
title = models.CharField(max_length=200)
|
||||
description = models.TextField(blank=True)
|
||||
@@ -27,47 +28,39 @@ class Gallery(BaseModel):
|
||||
return self.title
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
previous_image_name = get_image_previous_name(self, "image")
|
||||
current_image_name = self.image.name if self.image else None
|
||||
image_changed = previous_image_name != current_image_name
|
||||
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
if self.image:
|
||||
# Get file size
|
||||
self.file_size = self.image.size
|
||||
|
||||
# Get image dimensions
|
||||
with Image.open(self.image.path) as img:
|
||||
self.width, self.height = img.size
|
||||
|
||||
# Compress image if it's too large
|
||||
self.compress_image()
|
||||
|
||||
# Update fields without triggering save again
|
||||
Gallery.objects.filter(pk=self.pk).update(
|
||||
file_size=self.file_size,
|
||||
width=self.width,
|
||||
height=self.height
|
||||
|
||||
if image_changed and previous_image_name:
|
||||
delete_image_derivatives_by_name(
|
||||
self.image.storage if self.image else None,
|
||||
previous_image_name,
|
||||
"gallery",
|
||||
delete_original=True,
|
||||
)
|
||||
|
||||
def compress_image(self):
|
||||
"""Compress image if it's larger than 2MB or dimensions are too large"""
|
||||
if not self.image:
|
||||
if image_changed:
|
||||
Gallery.objects.filter(pk=self.pk).update(file_size=None, width=None, height=None)
|
||||
return
|
||||
|
||||
with Image.open(self.image.path) as img:
|
||||
# Convert to RGB if necessary
|
||||
if img.mode in ("RGBA", "P"):
|
||||
img = img.convert("RGB")
|
||||
|
||||
# Resize if too large
|
||||
max_size = (1920, 1080)
|
||||
if img.size[0] > max_size[0] or img.size[1] > max_size[1]:
|
||||
img.thumbnail(max_size, Image.Resampling.LANCZOS)
|
||||
|
||||
# Compress if file size is too large
|
||||
quality = 85
|
||||
if self.file_size and self.file_size > MAX_IMAGE_FILE_SIZE_BYTES:
|
||||
quality = 70
|
||||
|
||||
img.save(self.image.path, "JPEG", quality=quality, optimize=True)
|
||||
|
||||
if getattr(self, "_defer_image_processing", False):
|
||||
return
|
||||
|
||||
if image_changed:
|
||||
result = safe_process_public_image(self.image, "gallery")
|
||||
if result:
|
||||
Gallery.objects.filter(pk=self.pk).update(
|
||||
file_size=result.file_size,
|
||||
width=result.width,
|
||||
height=result.height,
|
||||
)
|
||||
self.file_size = result.file_size
|
||||
self.width = result.width
|
||||
self.height = result.height
|
||||
|
||||
@property
|
||||
def file_size_mb(self):
|
||||
|
||||
@@ -1,20 +1,26 @@
|
||||
from celery import shared_task
|
||||
from PIL import Image
|
||||
import logging
|
||||
|
||||
from celery import shared_task
|
||||
|
||||
from core.media import safe_process_public_image
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@shared_task
|
||||
def process_uploaded_image(gallery_id):
|
||||
"""Process uploaded image: compress, resize, extract metadata"""
|
||||
"""Process gallery image derivatives and refresh metadata."""
|
||||
try:
|
||||
from .models import Gallery
|
||||
gallery_item = Gallery.objects.get(id=gallery_id)
|
||||
|
||||
if gallery_item.image:
|
||||
# This will trigger the compression and metadata extraction
|
||||
gallery_item.compress_image()
|
||||
|
||||
result = safe_process_public_image(gallery_item.image, "gallery", force=True)
|
||||
if result:
|
||||
Gallery.objects.filter(pk=gallery_item.pk).update(
|
||||
file_size=result.file_size,
|
||||
width=result.width,
|
||||
height=result.height,
|
||||
)
|
||||
logger.info(f"Processed image: {gallery_item.title}")
|
||||
return f"Processed image: {gallery_item.title}"
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from ninja import Schema, ModelSchema
|
||||
from typing import Optional
|
||||
|
||||
from core.media import PREVIEW_VARIANT, THUMBNAIL_VARIANT, derivative_url
|
||||
from apps.users.models import User
|
||||
|
||||
|
||||
@@ -23,6 +24,8 @@ class UserLoginSchema(Schema):
|
||||
|
||||
class UserProfileSchema(ModelSchema):
|
||||
profile_picture: Optional[str] = None
|
||||
profile_picture_thumbnail_url: Optional[str] = None
|
||||
profile_picture_preview_url: Optional[str] = None
|
||||
student_id: Optional[str] = None
|
||||
major: Optional[str] = None
|
||||
university: Optional[str] = None
|
||||
@@ -68,6 +71,18 @@ class UserProfileSchema(ModelSchema):
|
||||
return request.build_absolute_uri(obj.profile_picture.url)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def resolve_profile_picture_thumbnail_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.profile_picture, THUMBNAIL_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_profile_picture_preview_url(obj, context):
|
||||
request = context["request"]
|
||||
url = derivative_url(obj.profile_picture, PREVIEW_VARIANT)
|
||||
return request.build_absolute_uri(url) if url else None
|
||||
|
||||
|
||||
class UserListSchema(ModelSchema):
|
||||
major: Optional[str] = None
|
||||
|
||||
@@ -5,13 +5,13 @@ from django.contrib.auth import authenticate
|
||||
from django.db.models import Q
|
||||
from django.shortcuts import get_object_or_404
|
||||
from django.utils import timezone
|
||||
from django.core.files.storage import default_storage
|
||||
from django.core.files.base import ContentFile
|
||||
|
||||
import uuid
|
||||
import jwt
|
||||
from ninja import Query, Router
|
||||
|
||||
from core.media import delete_image_derivatives
|
||||
from apps.users.models import User, Major, University
|
||||
from apps.users.tasks import send_verification_email, send_password_reset_email
|
||||
from apps.users.api.schemas import (
|
||||
@@ -254,10 +254,6 @@ def upload_profile_picture(request):
|
||||
|
||||
user = request.auth
|
||||
|
||||
# Delete old profile picture if exists
|
||||
if user.profile_picture:
|
||||
default_storage.delete(user.profile_picture.name)
|
||||
|
||||
# Save new profile picture
|
||||
filename = f"profile_pictures/{user.id}_{uuid.uuid4().hex}.{file.name.split('.')[-1]}"
|
||||
user.profile_picture.save(filename, ContentFile(file.read()))
|
||||
@@ -270,7 +266,7 @@ def delete_profile_picture(request):
|
||||
user = request.auth
|
||||
|
||||
if user.profile_picture:
|
||||
default_storage.delete(user.profile_picture.name)
|
||||
delete_image_derivatives(user.profile_picture, "profile_picture", delete_original=True)
|
||||
user.profile_picture = None
|
||||
user.save(update_fields=['profile_picture'])
|
||||
|
||||
@@ -400,4 +396,3 @@ def check_username_availability(request, username: str):
|
||||
"""Check if a username is available for registration"""
|
||||
exists = User.objects.filter(username=username).exists()
|
||||
return {"exists": exists}
|
||||
|
||||
|
||||
@@ -5,6 +5,11 @@ from django.db import models
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
|
||||
from core.media import (
|
||||
delete_image_derivatives_by_name,
|
||||
get_image_previous_name,
|
||||
safe_process_public_image,
|
||||
)
|
||||
from core.models import BaseModel
|
||||
|
||||
|
||||
@@ -95,6 +100,8 @@ class User(AbstractUser, BaseModel):
|
||||
self.save(update_fields=['password_reset_token', 'password_reset_token_expires_at'])
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
previous_image_name = get_image_previous_name(self, "profile_picture")
|
||||
current_image_name = self.profile_picture.name if self.profile_picture else None
|
||||
send_verified_success = False
|
||||
|
||||
if self.pk is not None:
|
||||
@@ -104,6 +111,17 @@ class User(AbstractUser, BaseModel):
|
||||
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
if previous_image_name != current_image_name and previous_image_name:
|
||||
delete_image_derivatives_by_name(
|
||||
self.profile_picture.storage if self.profile_picture else None,
|
||||
previous_image_name,
|
||||
"profile_picture",
|
||||
delete_original=True,
|
||||
)
|
||||
|
||||
if previous_image_name != current_image_name and self.profile_picture:
|
||||
safe_process_public_image(self.profile_picture, "profile_picture")
|
||||
|
||||
if send_verified_success:
|
||||
try:
|
||||
from apps.users.tasks import send_email_verified_success
|
||||
|
||||
1
core/management/__init__.py
Normal file
1
core/management/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
1
core/management/commands/__init__.py
Normal file
1
core/management/commands/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
71
core/management/commands/backfill_media_derivatives.py
Normal file
71
core/management/commands/backfill_media_derivatives.py
Normal file
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from apps.blog.models import Post
|
||||
from apps.events.models import Event
|
||||
from apps.gallery.models import Gallery
|
||||
from apps.users.models import User
|
||||
from core.media import safe_process_public_image
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackfillTarget:
|
||||
label: str
|
||||
queryset: object
|
||||
field_name: str
|
||||
family: str
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Generate missing public image derivatives for existing media."
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Regenerate derivatives even when sidecar files already exist.",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
force = options["force"]
|
||||
processed = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
|
||||
targets = (
|
||||
BackfillTarget("events", Event.objects.exclude(featured_image="").exclude(featured_image__isnull=True), "featured_image", "event_featured"),
|
||||
BackfillTarget("gallery", Gallery.objects.exclude(image="").exclude(image__isnull=True), "image", "gallery"),
|
||||
BackfillTarget("blog", Post.objects.exclude(featured_image="").exclude(featured_image__isnull=True), "featured_image", "blog_featured"),
|
||||
BackfillTarget("users", User.objects.exclude(profile_picture="").exclude(profile_picture__isnull=True), "profile_picture", "profile_picture"),
|
||||
)
|
||||
|
||||
for target in targets:
|
||||
self.stdout.write(self.style.NOTICE(f"Backfilling {target.label}..."))
|
||||
for instance in target.queryset.iterator():
|
||||
file_field = getattr(instance, target.field_name)
|
||||
result = safe_process_public_image(file_field, target.family, force=force)
|
||||
if result is None:
|
||||
failed += 1
|
||||
self.stdout.write(self.style.WARNING(f"failed: {target.label}#{instance.pk}"))
|
||||
continue
|
||||
|
||||
if hasattr(instance, "file_size") and hasattr(instance, "width") and hasattr(instance, "height"):
|
||||
type(instance).objects.filter(pk=instance.pk).update(
|
||||
file_size=result.file_size,
|
||||
width=result.width,
|
||||
height=result.height,
|
||||
)
|
||||
|
||||
if result.processed:
|
||||
processed += 1
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
f"Media derivative backfill finished. processed={processed} skipped={skipped} failed={failed}"
|
||||
)
|
||||
)
|
||||
239
core/media.py
Normal file
239
core/media.py
Normal file
@@ -0,0 +1,239 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from django.core.files.storage import Storage, default_storage
|
||||
|
||||
from PIL import Image, ImageFilter, ImageOps, UnidentifiedImageError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BLUR_VARIANT = "blur"
|
||||
PREVIEW_VARIANT = "preview"
|
||||
THUMBNAIL_VARIANT = "thumbnail"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageVariantSpec:
|
||||
key: str
|
||||
max_size: tuple[int, int]
|
||||
quality: int = 80
|
||||
blur_radius: float = 0.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageFamilySpec:
|
||||
key: str
|
||||
original_max_size: tuple[int, int]
|
||||
original_quality: int
|
||||
variants: tuple[ImageVariantSpec, ...]
|
||||
|
||||
|
||||
IMAGE_FAMILIES: dict[str, ImageFamilySpec] = {
|
||||
"event_featured": ImageFamilySpec(
|
||||
key="event_featured",
|
||||
original_max_size=(2560, 2560),
|
||||
original_quality=84,
|
||||
variants=(
|
||||
ImageVariantSpec(THUMBNAIL_VARIANT, (640, 640), quality=72),
|
||||
ImageVariantSpec(PREVIEW_VARIANT, (1600, 1600), quality=82),
|
||||
ImageVariantSpec(BLUR_VARIANT, (48, 48), quality=36, blur_radius=10.0),
|
||||
),
|
||||
),
|
||||
"gallery": ImageFamilySpec(
|
||||
key="gallery",
|
||||
original_max_size=(2560, 2560),
|
||||
original_quality=84,
|
||||
variants=(
|
||||
ImageVariantSpec(THUMBNAIL_VARIANT, (480, 480), quality=68),
|
||||
ImageVariantSpec(PREVIEW_VARIANT, (1280, 1280), quality=78),
|
||||
ImageVariantSpec(BLUR_VARIANT, (48, 48), quality=34, blur_radius=12.0),
|
||||
),
|
||||
),
|
||||
"blog_featured": ImageFamilySpec(
|
||||
key="blog_featured",
|
||||
original_max_size=(2560, 2560),
|
||||
original_quality=84,
|
||||
variants=(
|
||||
ImageVariantSpec(THUMBNAIL_VARIANT, (640, 640), quality=72),
|
||||
ImageVariantSpec(PREVIEW_VARIANT, (1600, 1600), quality=82),
|
||||
ImageVariantSpec(BLUR_VARIANT, (48, 48), quality=36, blur_radius=10.0),
|
||||
),
|
||||
),
|
||||
"profile_picture": ImageFamilySpec(
|
||||
key="profile_picture",
|
||||
original_max_size=(1200, 1200),
|
||||
original_quality=82,
|
||||
variants=(
|
||||
ImageVariantSpec(THUMBNAIL_VARIANT, (160, 160), quality=72),
|
||||
ImageVariantSpec(PREVIEW_VARIANT, (640, 640), quality=82),
|
||||
ImageVariantSpec(BLUR_VARIANT, (40, 40), quality=32, blur_radius=10.0),
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessedImageResult:
|
||||
width: int
|
||||
height: int
|
||||
file_size: int
|
||||
processed: bool
|
||||
|
||||
|
||||
def get_image_family_spec(family: str) -> ImageFamilySpec:
|
||||
try:
|
||||
return IMAGE_FAMILIES[family]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"Unknown image family: {family}") from exc
|
||||
|
||||
|
||||
def get_image_previous_name(instance, field_name: str) -> str | None:
|
||||
if not getattr(instance, "pk", None):
|
||||
return None
|
||||
|
||||
manager = getattr(instance.__class__, "all_objects", instance.__class__._default_manager)
|
||||
return manager.filter(pk=instance.pk).values_list(field_name, flat=True).first()
|
||||
|
||||
|
||||
def derivative_name(original_name: str, variant_key: str) -> str:
|
||||
original_path = Path(original_name)
|
||||
return str(original_path.with_name(f"{original_path.stem}.{variant_key}.webp")).replace("\\", "/")
|
||||
|
||||
|
||||
def iter_derivative_names(original_name: str, family: str) -> Iterable[str]:
|
||||
for variant in get_image_family_spec(family).variants:
|
||||
yield derivative_name(original_name, variant.key)
|
||||
|
||||
|
||||
def derivative_url(file_field, variant_key: str) -> str | None:
|
||||
if not file_field or not getattr(file_field, "name", None):
|
||||
return None
|
||||
|
||||
name = derivative_name(file_field.name, variant_key)
|
||||
if not file_field.storage.exists(name):
|
||||
return None
|
||||
|
||||
return file_field.storage.url(name)
|
||||
|
||||
|
||||
def delete_image_derivatives_by_name(
|
||||
storage: Storage | None,
|
||||
original_name: str | None,
|
||||
family: str,
|
||||
*,
|
||||
delete_original: bool = False,
|
||||
) -> None:
|
||||
if not original_name:
|
||||
return
|
||||
|
||||
storage = storage or default_storage
|
||||
for name in iter_derivative_names(original_name, family):
|
||||
if storage.exists(name):
|
||||
storage.delete(name)
|
||||
|
||||
if delete_original and storage.exists(original_name):
|
||||
storage.delete(original_name)
|
||||
|
||||
|
||||
def delete_image_derivatives(file_field, family: str, *, delete_original: bool = False) -> None:
|
||||
if not file_field or not getattr(file_field, "name", None):
|
||||
return
|
||||
delete_image_derivatives_by_name(
|
||||
getattr(file_field, "storage", default_storage),
|
||||
file_field.name,
|
||||
family,
|
||||
delete_original=delete_original,
|
||||
)
|
||||
|
||||
|
||||
def process_public_image(file_field, family: str, *, force: bool = False) -> ProcessedImageResult:
|
||||
if not file_field or not getattr(file_field, "path", None):
|
||||
raise ValueError("Image field must point to a local file path.")
|
||||
|
||||
spec = get_image_family_spec(family)
|
||||
missing_derivatives = [
|
||||
name
|
||||
for name in iter_derivative_names(file_field.name, family)
|
||||
if not file_field.storage.exists(name)
|
||||
]
|
||||
should_process = force or bool(missing_derivatives)
|
||||
|
||||
if should_process:
|
||||
with Image.open(file_field.path) as opened:
|
||||
source = ImageOps.exif_transpose(opened)
|
||||
optimized = _resize_for_original(source, spec.original_max_size)
|
||||
_save_original(optimized, file_field.path, quality=spec.original_quality)
|
||||
|
||||
for variant in spec.variants:
|
||||
derivative = _build_variant_image(optimized, variant)
|
||||
target_name = derivative_name(file_field.name, variant.key)
|
||||
target_path = file_field.storage.path(target_name)
|
||||
Path(target_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
derivative.save(target_path, format="WEBP", quality=variant.quality, method=6)
|
||||
|
||||
with Image.open(file_field.path) as current:
|
||||
width, height = current.size
|
||||
|
||||
file_size = file_field.storage.size(file_field.name)
|
||||
return ProcessedImageResult(
|
||||
width=width,
|
||||
height=height,
|
||||
file_size=file_size,
|
||||
processed=should_process,
|
||||
)
|
||||
|
||||
|
||||
def _resize_for_original(image: Image.Image, max_size: tuple[int, int]) -> Image.Image:
|
||||
prepared = image.copy()
|
||||
prepared.thumbnail(max_size, Image.Resampling.LANCZOS)
|
||||
return prepared
|
||||
|
||||
|
||||
def _save_original(image: Image.Image, path: str, *, quality: int) -> None:
|
||||
suffix = Path(path).suffix.lower()
|
||||
has_alpha = "A" in image.getbands()
|
||||
|
||||
if suffix in {".jpg", ".jpeg"}:
|
||||
image.convert("RGB").save(
|
||||
path,
|
||||
format="JPEG",
|
||||
quality=quality,
|
||||
optimize=True,
|
||||
progressive=True,
|
||||
)
|
||||
return
|
||||
|
||||
if suffix == ".png":
|
||||
target = image if has_alpha else image.convert("RGB")
|
||||
target.save(path, format="PNG", optimize=True, compress_level=9)
|
||||
return
|
||||
|
||||
if suffix == ".webp":
|
||||
image.save(path, format="WEBP", quality=quality, method=6)
|
||||
return
|
||||
|
||||
image.save(path)
|
||||
|
||||
|
||||
def _build_variant_image(image: Image.Image, variant: ImageVariantSpec) -> Image.Image:
|
||||
target = image.copy()
|
||||
if variant.blur_radius > 0:
|
||||
target.thumbnail(variant.max_size, Image.Resampling.LANCZOS)
|
||||
target = target.filter(ImageFilter.GaussianBlur(radius=variant.blur_radius))
|
||||
return target
|
||||
|
||||
target.thumbnail(variant.max_size, Image.Resampling.LANCZOS)
|
||||
return target
|
||||
|
||||
|
||||
def safe_process_public_image(file_field, family: str, *, force: bool = False) -> ProcessedImageResult | None:
|
||||
try:
|
||||
return process_public_image(file_field, family, force=force)
|
||||
except (FileNotFoundError, OSError, UnidentifiedImageError, ValueError) as exc:
|
||||
logger.warning("Failed to process image '%s': %s", getattr(file_field, "name", None), exc)
|
||||
return None
|
||||
1
core/tests/__init__.py
Normal file
1
core/tests/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
1
core/tests/unit/__init__.py
Normal file
1
core/tests/unit/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
174
core/tests/unit/test_media.py
Normal file
174
core/tests/unit/test_media.py
Normal file
@@ -0,0 +1,174 @@
|
||||
import io
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import timedelta
|
||||
|
||||
from PIL import Image
|
||||
from django.core.management import call_command
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.test import RequestFactory
|
||||
from django.test import TestCase, override_settings
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.blog.api.schemas import AuthorSchema, PostListSchema
|
||||
from apps.blog.models import Post
|
||||
from apps.events.api.schemas import EventGallerySchema, EventListSchema
|
||||
from apps.events.models import Event
|
||||
from apps.gallery.api.schemas import GallerySchema
|
||||
from apps.gallery.models import Gallery
|
||||
from apps.users.api.schemas import UserProfileSchema
|
||||
from apps.users.models import User
|
||||
from core.media import (
|
||||
BLUR_VARIANT,
|
||||
PREVIEW_VARIANT,
|
||||
THUMBNAIL_VARIANT,
|
||||
delete_image_derivatives,
|
||||
derivative_name,
|
||||
process_public_image,
|
||||
)
|
||||
|
||||
|
||||
class MediaProcessingTests(TestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.media_root = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(self.media_root, ignore_errors=True))
|
||||
self.override = override_settings(MEDIA_ROOT=self.media_root, MEDIA_URL="/media/")
|
||||
self.override.enable()
|
||||
self.addCleanup(self.override.disable)
|
||||
self.user = User.objects.create_user(
|
||||
username="media-user",
|
||||
email="media@example.com",
|
||||
password="TestPass123!",
|
||||
)
|
||||
self.factory = RequestFactory()
|
||||
|
||||
def _upload(self, name: str, *, mode: str = "RGB", fmt: str = "JPEG") -> SimpleUploadedFile:
|
||||
buffer = io.BytesIO()
|
||||
Image.new(mode, (2400, 1800), color=(20, 80, 200)).save(buffer, format=fmt)
|
||||
buffer.seek(0)
|
||||
return SimpleUploadedFile(name, buffer.read(), content_type=f"image/{fmt.lower()}")
|
||||
|
||||
def test_process_public_image_generates_derivatives_for_jpeg(self):
|
||||
gallery = Gallery.objects.create(
|
||||
title="JPEG image",
|
||||
uploaded_by=self.user,
|
||||
image=self._upload("photo.jpg", fmt="JPEG"),
|
||||
)
|
||||
|
||||
result = process_public_image(gallery.image, "gallery", force=True)
|
||||
|
||||
self.assertTrue(result.processed)
|
||||
self.assertTrue(gallery.image.storage.exists(derivative_name(gallery.image.name, THUMBNAIL_VARIANT)))
|
||||
self.assertTrue(gallery.image.storage.exists(derivative_name(gallery.image.name, PREVIEW_VARIANT)))
|
||||
self.assertTrue(gallery.image.storage.exists(derivative_name(gallery.image.name, BLUR_VARIANT)))
|
||||
|
||||
def test_process_public_image_generates_derivatives_for_png(self):
|
||||
gallery = Gallery.objects.create(
|
||||
title="PNG image",
|
||||
uploaded_by=self.user,
|
||||
image=self._upload("photo.png", mode="RGBA", fmt="PNG"),
|
||||
)
|
||||
|
||||
result = process_public_image(gallery.image, "gallery", force=True)
|
||||
|
||||
self.assertEqual(result.width, 2400)
|
||||
self.assertEqual(result.height, 1800)
|
||||
self.assertTrue(gallery.image.storage.exists(derivative_name(gallery.image.name, PREVIEW_VARIANT)))
|
||||
|
||||
def test_delete_image_derivatives_removes_sidecars(self):
|
||||
gallery = Gallery.objects.create(
|
||||
title="Delete derivatives",
|
||||
uploaded_by=self.user,
|
||||
image=self._upload("cleanup.jpg", fmt="JPEG"),
|
||||
)
|
||||
process_public_image(gallery.image, "gallery", force=True)
|
||||
|
||||
delete_image_derivatives(gallery.image, "gallery")
|
||||
|
||||
self.assertFalse(gallery.image.storage.exists(derivative_name(gallery.image.name, THUMBNAIL_VARIANT)))
|
||||
self.assertFalse(gallery.image.storage.exists(derivative_name(gallery.image.name, PREVIEW_VARIANT)))
|
||||
self.assertFalse(gallery.image.storage.exists(derivative_name(gallery.image.name, BLUR_VARIANT)))
|
||||
|
||||
def test_replacing_image_removes_previous_sidecars(self):
|
||||
gallery = Gallery.objects.create(
|
||||
title="Replace derivatives",
|
||||
uploaded_by=self.user,
|
||||
image=self._upload("replace-first.jpg", fmt="JPEG"),
|
||||
)
|
||||
first_name = gallery.image.name
|
||||
|
||||
gallery.image = self._upload("replace-second.jpg", fmt="JPEG")
|
||||
gallery.save()
|
||||
|
||||
self.assertFalse(gallery.image.storage.exists(derivative_name(first_name, THUMBNAIL_VARIANT)))
|
||||
self.assertFalse(gallery.image.storage.exists(derivative_name(first_name, PREVIEW_VARIANT)))
|
||||
self.assertFalse(gallery.image.storage.exists(derivative_name(first_name, BLUR_VARIANT)))
|
||||
|
||||
def test_backfill_command_is_idempotent(self):
|
||||
gallery = Gallery(title="Backfill image", uploaded_by=self.user)
|
||||
gallery._defer_image_processing = True
|
||||
gallery.image = self._upload("backfill.jpg", fmt="JPEG")
|
||||
gallery.save()
|
||||
|
||||
first = io.StringIO()
|
||||
second = io.StringIO()
|
||||
call_command("backfill_media_derivatives", stdout=first)
|
||||
call_command("backfill_media_derivatives", stdout=second)
|
||||
|
||||
self.assertIn("processed=1", first.getvalue())
|
||||
self.assertIn("skipped=1", second.getvalue())
|
||||
|
||||
def test_schema_resolvers_expose_derivative_urls(self):
|
||||
self.user.profile_picture.save("profile.jpg", self._upload("profile.jpg", fmt="JPEG"))
|
||||
|
||||
gallery = Gallery.objects.create(
|
||||
title="Schema image",
|
||||
uploaded_by=self.user,
|
||||
image=self._upload("schema-gallery.jpg", fmt="JPEG"),
|
||||
)
|
||||
event = Event.objects.create(
|
||||
title="Schema event",
|
||||
description="Event description",
|
||||
start_time=timezone.now(),
|
||||
end_time=timezone.now() + timedelta(hours=2),
|
||||
featured_image=self._upload("schema-event.jpg", fmt="JPEG"),
|
||||
)
|
||||
post = Post.objects.create(
|
||||
title="Schema post",
|
||||
content="Post body",
|
||||
author=self.user,
|
||||
featured_image=self._upload("schema-post.jpg", fmt="JPEG"),
|
||||
status=Post.StatusChoices.PUBLISHED,
|
||||
)
|
||||
|
||||
context = {"request": self.factory.get("/")}
|
||||
|
||||
self.assertIn(
|
||||
derivative_name(event.featured_image.name, THUMBNAIL_VARIANT),
|
||||
EventListSchema.resolve_absolute_featured_image_thumbnail_url(event, context),
|
||||
)
|
||||
self.assertIn(
|
||||
derivative_name(event.featured_image.name, PREVIEW_VARIANT),
|
||||
EventListSchema.resolve_absolute_featured_image_preview_url(event, context),
|
||||
)
|
||||
self.assertIn(
|
||||
derivative_name(gallery.image.name, PREVIEW_VARIANT),
|
||||
EventGallerySchema.resolve_absolute_image_preview_url(gallery, context),
|
||||
)
|
||||
self.assertIn(
|
||||
derivative_name(gallery.image.name, BLUR_VARIANT),
|
||||
GallerySchema.resolve_absolute_image_blur_url(gallery, context),
|
||||
)
|
||||
self.assertIn(
|
||||
derivative_name(post.featured_image.name, THUMBNAIL_VARIANT),
|
||||
PostListSchema.resolve_absolute_featured_image_thumbnail_url(post, context),
|
||||
)
|
||||
self.assertIn(
|
||||
derivative_name(self.user.profile_picture.name, THUMBNAIL_VARIANT),
|
||||
AuthorSchema.resolve_profile_picture_thumbnail_url(self.user, context),
|
||||
)
|
||||
self.assertIn(
|
||||
derivative_name(self.user.profile_picture.name, PREVIEW_VARIANT),
|
||||
UserProfileSchema.resolve_profile_picture_preview_url(self.user, context),
|
||||
)
|
||||
Reference in New Issue
Block a user