From 30169a3a270bd9c42e612e90f0cce037bfe14d3e Mon Sep 17 00:00:00 2001 From: benya Date: Sun, 8 Mar 2026 12:40:49 +0300 Subject: [PATCH] feat: add waveform voice messages end-to-end --- .../versions/0020_attachment_waveform_data.py | 27 ++++ app/media/models.py | 3 +- app/media/repository.py | 14 ++ app/media/schemas.py | 3 + app/media/service.py | 44 +++++- app/messages/schemas.py | 1 + app/messages/service.py | 24 +++ docs/api-reference.md | 7 +- web/src/api/chats.ts | 11 +- web/src/chat/types.ts | 2 + web/src/components/MessageComposer.tsx | 144 +++++++++++++++++- web/src/components/MessageList.tsx | 99 +++++++++++- web/src/utils/formatMessage.tsx | 1 + 13 files changed, 361 insertions(+), 19 deletions(-) create mode 100644 alembic/versions/0020_attachment_waveform_data.py diff --git a/alembic/versions/0020_attachment_waveform_data.py b/alembic/versions/0020_attachment_waveform_data.py new file mode 100644 index 0000000..7017fd7 --- /dev/null +++ b/alembic/versions/0020_attachment_waveform_data.py @@ -0,0 +1,27 @@ +"""add waveform data to attachments + +Revision ID: 0020_attachment_waveform_data +Revises: 0019_user_privacy_fields +Create Date: 2026-03-08 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "0020_attachment_waveform_data" +down_revision: str | None = "0019_user_privacy_fields" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column("attachments", sa.Column("waveform_data", sa.Text(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("attachments", "waveform_data") + diff --git a/app/media/models.py b/app/media/models.py index 48edb0d..9c2d344 100644 --- a/app/media/models.py +++ b/app/media/models.py @@ -1,4 +1,4 @@ -from sqlalchemy import ForeignKey, Integer, String +from sqlalchemy import ForeignKey, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column, relationship from app.database.base import Base @@ -12,5 +12,6 @@ class Attachment(Base): file_url: Mapped[str] = mapped_column(String(1024), nullable=False) file_type: Mapped[str] = mapped_column(String(64), nullable=False) file_size: Mapped[int] = mapped_column(Integer, nullable=False) + waveform_data: Mapped[str | None] = mapped_column(Text, nullable=True) message = relationship("Message", back_populates="attachments") diff --git a/app/media/repository.py b/app/media/repository.py index 0c96844..95701e5 100644 --- a/app/media/repository.py +++ b/app/media/repository.py @@ -12,12 +12,14 @@ async def create_attachment( file_url: str, file_type: str, file_size: int, + waveform_data: str | None = None, ) -> Attachment: attachment = Attachment( message_id=message_id, file_url=file_url, file_type=file_type, file_size=file_size, + waveform_data=waveform_data, ) db.add(attachment) await db.flush() @@ -46,3 +48,15 @@ async def list_chat_attachments( stmt = stmt.where(Attachment.id < before_id) result = await db.execute(stmt) return [(row[0], row[1]) for row in result.all()] + + +async def list_attachments_by_message_ids( + db: AsyncSession, + *, + message_ids: list[int], +) -> list[Attachment]: + if not message_ids: + return [] + stmt = select(Attachment).where(Attachment.message_id.in_(message_ids)) + result = await db.execute(stmt) + return list(result.scalars().all()) diff --git a/app/media/schemas.py b/app/media/schemas.py index 460fb13..86ce349 100644 --- a/app/media/schemas.py +++ b/app/media/schemas.py @@ -21,6 +21,7 @@ class AttachmentCreateRequest(BaseModel): file_url: str = Field(min_length=1, max_length=1024) file_type: str = Field(min_length=1, max_length=64) file_size: int = Field(gt=0) + waveform_points: list[int] | None = Field(default=None, min_length=8, max_length=256) class AttachmentRead(BaseModel): @@ -31,6 +32,7 @@ class AttachmentRead(BaseModel): file_url: str file_type: str file_size: int + waveform_points: list[int] | None = None class ChatAttachmentRead(BaseModel): @@ -42,3 +44,4 @@ class ChatAttachmentRead(BaseModel): file_url: str file_type: str file_size: int + waveform_points: list[int] | None = None diff --git a/app/media/service.py b/app/media/service.py index ff94830..b848bf9 100644 --- a/app/media/service.py +++ b/app/media/service.py @@ -1,4 +1,5 @@ import mimetypes +import json from urllib.parse import quote from uuid import uuid4 @@ -29,6 +30,35 @@ ALLOWED_MIME_TYPES = { "text/plain", } + +def _normalize_waveform(points: list[int] | None) -> list[int] | None: + if points is None: + return None + normalized = [max(0, min(31, int(value))) for value in points] + if len(normalized) < 8: + return None + return normalized + + +def _decode_waveform(raw: str | None) -> list[int] | None: + if not raw: + return None + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return None + if not isinstance(parsed, list): + return None + result: list[int] = [] + for value in parsed[:256]: + if isinstance(value, int): + result.append(max(0, min(31, value))) + elif isinstance(value, float): + result.append(max(0, min(31, int(value)))) + else: + return None + return result or None + def _normalize_mime(file_type: str) -> str: return file_type.split(";", maxsplit=1)[0].strip().lower() @@ -125,16 +155,27 @@ async def store_attachment_metadata( detail="Only the message sender can attach files", ) + normalized_waveform = _normalize_waveform(payload.waveform_points) attachment = await repository.create_attachment( db, message_id=payload.message_id, file_url=payload.file_url, file_type=payload.file_type, file_size=payload.file_size, + waveform_data=json.dumps(normalized_waveform, ensure_ascii=True) if normalized_waveform else None, ) await db.commit() await db.refresh(attachment) - return AttachmentRead.model_validate(attachment) + return AttachmentRead.model_validate( + { + "id": attachment.id, + "message_id": attachment.message_id, + "file_url": attachment.file_url, + "file_type": attachment.file_type, + "file_size": attachment.file_size, + "waveform_points": _decode_waveform(attachment.waveform_data), + } + ) async def list_attachments_for_chat( @@ -162,6 +203,7 @@ async def list_attachments_for_chat( file_url=attachment.file_url, file_type=attachment.file_type, file_size=attachment.file_size, + waveform_points=_decode_waveform(attachment.waveform_data), ) for attachment, message in rows ] diff --git a/app/messages/schemas.py b/app/messages/schemas.py index ca9a71d..c65ebbb 100644 --- a/app/messages/schemas.py +++ b/app/messages/schemas.py @@ -17,6 +17,7 @@ class MessageRead(BaseModel): type: MessageType text: str | None delivery_status: Literal["sending", "sent", "delivered", "read"] | None = None + attachment_waveform: list[int] | None = None created_at: datetime updated_at: datetime diff --git a/app/messages/service.py b/app/messages/service.py index 4e286cd..facc4d0 100644 --- a/app/messages/service.py +++ b/app/messages/service.py @@ -1,3 +1,5 @@ +import json + from fastapi import HTTPException, status from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -5,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.chats import repository as chats_repository from app.chats.models import ChatMemberRole, ChatType from app.chats.service import ensure_chat_membership +from app.media import repository as media_repository from app.messages import repository from app.messages.models import Message from app.messages.spam_guard import enforce_message_spam_policy @@ -110,11 +113,32 @@ async def get_messages( messages = await repository.list_chat_messages(db, chat_id, user_id=user_id, limit=safe_limit, before_id=before_id) if not messages: return messages + message_ids = [message.id for message in messages] + attachments = await media_repository.list_attachments_by_message_ids(db, message_ids=message_ids) + waveform_by_message_id: dict[int, list[int]] = {} + for attachment in attachments: + if not attachment.waveform_data: + continue + try: + parsed = json.loads(attachment.waveform_data) + except Exception: + continue + if not isinstance(parsed, list): + continue + values: list[int] = [] + for item in parsed[:256]: + if isinstance(item, (int, float)): + values.append(max(0, min(31, int(item)))) + if values: + waveform_by_message_id[attachment.message_id] = values receipts = await repository.list_chat_receipts(db, chat_id=chat_id) other_receipts = [receipt for receipt in receipts if receipt.user_id != user_id] if not other_receipts: return messages for message in messages: + waveform = waveform_by_message_id.get(message.id) + if waveform: + setattr(message, "attachment_waveform", waveform) if message.sender_id != user_id: continue is_read = any((receipt.last_read_message_id or 0) >= message.id for receipt in other_receipts) diff --git a/docs/api-reference.md b/docs/api-reference.md index 88f8967..101244c 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -255,6 +255,7 @@ Rules: "type": "text", "text": "Hello", "delivery_status": "read", + "attachment_waveform": [4, 7, 10, 9, 6], "created_at": "2026-03-08T10:02:00Z", "updated_at": "2026-03-08T10:02:00Z" } @@ -339,7 +340,8 @@ Rules: "message_id": 100, "file_url": "https://.../bucket/uploads/....jpg", "file_type": "image/jpeg", - "file_size": 123456 + "file_size": 123456, + "waveform_points": [4, 7, 10, 9, 6] } ``` @@ -351,7 +353,8 @@ Rules: "message_id": 100, "file_url": "https://...", "file_type": "image/jpeg", - "file_size": 123456 + "file_size": 123456, + "waveform_points": [4, 7, 10, 9, 6] } ``` diff --git a/web/src/api/chats.ts b/web/src/api/chats.ts index c92c6c5..c75043b 100644 --- a/web/src/api/chats.ts +++ b/web/src/api/chats.ts @@ -147,12 +147,19 @@ export async function uploadToPresignedUrl( } } -export async function attachFile(messageId: number, fileUrl: string, fileType: string, fileSize: number): Promise { +export async function attachFile( + messageId: number, + fileUrl: string, + fileType: string, + fileSize: number, + waveformPoints?: number[] | null +): Promise { await http.post("/media/attachments", { message_id: messageId, file_url: fileUrl, file_type: fileType, - file_size: fileSize + file_size: fileSize, + waveform_points: waveformPoints ?? undefined }); } diff --git a/web/src/chat/types.ts b/web/src/chat/types.ts index 3f3a233..fb1b17c 100644 --- a/web/src/chat/types.ts +++ b/web/src/chat/types.ts @@ -62,6 +62,7 @@ export interface Message { client_message_id?: string; delivery_status?: DeliveryStatus; is_pending?: boolean; + attachment_waveform?: number[] | null; } export interface MessageReaction { @@ -123,4 +124,5 @@ export interface ChatAttachment { file_url: string; file_type: string; file_size: number; + waveform_points?: number[] | null; } diff --git a/web/src/components/MessageComposer.tsx b/web/src/components/MessageComposer.tsx index b7ba57f..5aa8dc3 100644 --- a/web/src/components/MessageComposer.tsx +++ b/web/src/components/MessageComposer.tsx @@ -41,9 +41,11 @@ export function MessageComposer() { const [selectedType, setSelectedType] = useState<"file" | "image" | "video" | "audio">("file"); const [previewUrl, setPreviewUrl] = useState(null); const [showAttachMenu, setShowAttachMenu] = useState(false); + const [showFormatMenu, setShowFormatMenu] = useState(false); const [captionDraft, setCaptionDraft] = useState(""); const mediaInputRef = useRef(null); const fileInputRef = useRef(null); + const textareaRef = useRef(null); const [recordingState, setRecordingState] = useState("idle"); const [recordSeconds, setRecordSeconds] = useState(0); @@ -169,7 +171,11 @@ export function MessageComposer() { } } - async function handleUpload(file: File, messageType: "file" | "image" | "video" | "audio" | "voice" = "file") { + async function handleUpload( + file: File, + messageType: "file" | "image" | "video" | "audio" | "voice" = "file", + waveformPoints?: number[] | null + ) { if (!activeChatId || !me) { return; } @@ -186,12 +192,19 @@ export function MessageComposer() { senderId: me.id, type: messageType, text: upload.file_url, - clientMessageId + clientMessageId, }); const message = await sendMessageWithClientId(activeChatId, upload.file_url, messageType, clientMessageId, replyToMessageId); - confirmMessageByClientId(activeChatId, clientMessageId, message); + const messageWithWaveform = waveformPoints?.length ? { ...message, attachment_waveform: waveformPoints } : message; + confirmMessageByClientId(activeChatId, clientMessageId, messageWithWaveform); try { - await attachFile(message.id, upload.file_url, file.type || "application/octet-stream", file.size); + await attachFile( + message.id, + upload.file_url, + file.type || "application/octet-stream", + file.size, + waveformPoints ?? null + ); } catch { setUploadError("File sent, but metadata save failed. Please refresh chat."); } @@ -297,7 +310,8 @@ export function MessageComposer() { } const blob = new Blob(data, { type: "audio/webm" }); const file = new File([blob], `voice-${Date.now()}.webm`, { type: "audio/webm" }); - await handleUpload(file, "voice"); + const waveform = await buildWaveformPoints(blob, 64); + await handleUpload(file, "voice", waveform); }; recorderRef.current = recorder; recorder.start(); @@ -453,6 +467,58 @@ export function MessageComposer() { return `${(size / (1024 * 1024)).toFixed(1)} MB`; } + function insertFormatting(startTag: string, endTag = startTag, placeholder = "text") { + const textarea = textareaRef.current; + if (!textarea) { + return; + } + const start = textarea.selectionStart ?? text.length; + const end = textarea.selectionEnd ?? text.length; + const selected = text.slice(start, end); + const middle = selected || placeholder; + const nextValue = `${text.slice(0, start)}${startTag}${middle}${endTag}${text.slice(end)}`; + setText(nextValue); + if (activeChatId) { + setDraft(activeChatId, nextValue); + } + requestAnimationFrame(() => { + textarea.focus(); + if (selected) { + const pos = start + startTag.length + middle.length + endTag.length; + textarea.setSelectionRange(pos, pos); + } else { + const selStart = start + startTag.length; + const selEnd = selStart + middle.length; + textarea.setSelectionRange(selStart, selEnd); + } + }); + } + + function insertLink() { + const textarea = textareaRef.current; + if (!textarea) { + return; + } + const start = textarea.selectionStart ?? text.length; + const end = textarea.selectionEnd ?? text.length; + const selected = text.slice(start, end).trim() || "text"; + const href = window.prompt("Enter URL (https://...)"); + if (!href) { + return; + } + const link = `[${selected}](${href.trim()})`; + const nextValue = `${text.slice(0, start)}${link}${text.slice(end)}`; + setText(nextValue); + if (activeChatId) { + setDraft(activeChatId, nextValue); + } + requestAnimationFrame(() => { + textarea.focus(); + const pos = start + link.length; + textarea.setSelectionRange(pos, pos); + }); + } + return (
{activeChatId && replyToByChat[activeChatId] ? ( @@ -492,6 +558,32 @@ export function MessageComposer() {
) : null} + {showFormatMenu ? ( +
+ + + + + + + +
+ ) : null} +
+ +