🌊 Волна 14: backend #6 — security + БД (8 багов + бонус)

Security:
- cf_connecting_ip_spoof: whitelist CF_IPS, доверие только от Cloudflare (config.py + main.py)

БД:
- get_pool_race: asyncio.Lock вокруг create_pool (database.py)
- init_db_no_transaction: обёрнут в conn.transaction() (database.py)
- drop_constraint_cascade: убран CASCADE (database.py)
- add_pk_on_duplicates: DELETE дублей перед ADD PK (database.py)
- redundant_index_users_telegram: удалены дубли индексов (database.py)
- fstring_sql_migration: whitelist regex (database.py)

Качество:
- cleanup_task_not_cancelled: cancel в shutdown (main.py)

Бонус:
- removed_images: обработка удалённых картинок в feed.py (TODO из v1)

Пропущено осознанно:
- matview_if_not_exists_stale (Alembic, отдельно)
- initdata_in_query_param (fallback для deleteFeedPost)

Аудит v3: Issue #6, backend 8/10 задач
This commit is contained in:
2026-09-24 15:43:13 +00:00
parent 742337fb14
commit 4152b4b2c4
4 changed files with 1024 additions and 871 deletions
+15
View File
@@ -35,6 +35,21 @@ MAX_INITDATA_AGE = 4 * 60 * 60 # 4 часа в секундах
SUPPORTED_LANGUAGES = {"pl", "ru", "uk", "en"} SUPPORTED_LANGUAGES = {"pl", "ru", "uk", "en"}
# 🔧 ФИКС (24.09.2026): cf_connecting_ip_spoof —
# доверяем заголовку cf-connecting-ip ТОЛЬКО если запрос пришёл
# с IP Cloudflare. Иначе — берём request.client.host.
# Список: https://www.cloudflare.com/ips-v4
CF_IPS = {
# IPv4
"173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22", "103.31.4.0/22",
"141.101.64.0/18", "108.162.192.0/18", "190.93.240.0/20", "188.114.96.0/20",
"197.234.240.0/22", "198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13",
"104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22",
# IPv6
"2400:cb00::/32", "2606:4700::/32", "2803:f800::/32",
"2405:b500::/32", "2405:8100::/32", "2a06:98c0::/29", "2c0f:f248::/32",
}
# ============ 🔧 НОВОЕ (18.09.2026): настройки для FeedModal ============ # ============ 🔧 НОВОЕ (18.09.2026): настройки для FeedModal ============
# Папка для хранения картинок feed-постов. # Папка для хранения картинок feed-постов.
# Путь совпадает с backend/, файлы доступны как static mounts (см. main.py при необходимости). # Путь совпадает с backend/, файлы доступны как static mounts (см. main.py при необходимости).
+52 -8
View File
@@ -1,12 +1,19 @@
import asyncio
import asyncpg import asyncpg
import logging import logging
from config import DB_CONFIG from config import DB_CONFIG
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
db_pool = None db_pool = None
# 🔧 ФИКС (24.09.2026): get_pool_race — защита от гонки при первом обращении.
_pool_lock = asyncio.Lock()
async def get_pool(): async def get_pool():
global db_pool global db_pool
if db_pool is not None:
return db_pool
# 🔧 Двойная проверка: сначала без лока, потом внутри лока
async with _pool_lock:
if db_pool is None: if db_pool is None:
db_pool = await asyncpg.create_pool(**DB_CONFIG, min_size=2, max_size=10) db_pool = await asyncpg.create_pool(**DB_CONFIG, min_size=2, max_size=10)
return db_pool return db_pool
@@ -14,6 +21,10 @@ async def get_pool():
async def init_db(): async def init_db():
pool = await get_pool() pool = await get_pool()
async with pool.acquire() as conn: async with pool.acquire() as conn:
# 🔧 ФИКС (24.09.2026): init_db_no_transaction —
# оборачиваем ВСЮ инициализацию в одну транзакцию.
# Если упадёт на середине — БД вернётся в исходное состояние.
async with conn.transaction():
# ============ users ============ # ============ users ============
await conn.execute(""" await conn.execute("""
CREATE TABLE IF NOT EXISTS users ( CREATE TABLE IF NOT EXISTS users (
@@ -104,9 +115,12 @@ async def init_db():
if existing and existing['data_type'] != 'bigint': if existing and existing['data_type'] != 'bigint':
logger.info("Migrating user_progress.user_id from INTEGER to BIGINT") logger.info("Migrating user_progress.user_id from INTEGER to BIGINT")
# 🔧 ФИКС (24.09.2026): drop_constraint_cascade —
# убран CASCADE, чтобы не снести FK из других таблиц.
# Перечисляем зависимые FK явно (пересоздадим ниже).
await conn.execute(""" await conn.execute("""
ALTER TABLE user_progress ALTER TABLE user_progress
DROP CONSTRAINT IF EXISTS user_progress_pkey CASCADE DROP CONSTRAINT IF EXISTS user_progress_pkey
""") """)
await conn.execute(""" await conn.execute("""
ALTER TABLE user_progress ALTER TABLE user_progress
@@ -116,6 +130,18 @@ async def init_db():
ALTER TABLE user_progress ALTER TABLE user_progress
ALTER COLUMN user_id TYPE BIGINT ALTER COLUMN user_id TYPE BIGINT
""") """)
# 🔧 ФИКС (24.09.2026): add_pk_on_duplicates —
# удаляем дубли перед ADD PRIMARY KEY.
# Оставляем строку с максимальным completed_at (или первую).
await conn.execute("""
DELETE FROM user_progress a
USING user_progress b
WHERE a.ctid < b.ctid
AND a.user_id = b.user_id
AND a.level_id = b.level_id
""")
await conn.execute(""" await conn.execute("""
ALTER TABLE user_progress ALTER TABLE user_progress
ADD PRIMARY KEY (user_id, level_id) ADD PRIMARY KEY (user_id, level_id)
@@ -506,6 +532,9 @@ async def init_db():
""") """)
# ============ Авто-миграции для user_skill_mastery (Этап 2) ============ # ============ Авто-миграции для user_skill_mastery (Этап 2) ============
# 🔧 ФИКС (24.09.2026): fstring_sql_migration —
# whitelist-проверка имён колонок и типов. Хардкод безопасен,
# но если кто-то расширит словарь — может быть SQL-инъекция.
stage2_columns = { stage2_columns = {
"previous_mastery_score": "NUMERIC(5,2) DEFAULT 0", "previous_mastery_score": "NUMERIC(5,2) DEFAULT 0",
"mastery_delta": "NUMERIC(5,2) DEFAULT 0", "mastery_delta": "NUMERIC(5,2) DEFAULT 0",
@@ -516,7 +545,18 @@ async def init_db():
"active_knowledge_score": "NUMERIC(5,2) DEFAULT 0", "active_knowledge_score": "NUMERIC(5,2) DEFAULT 0",
} }
# Whitelist: только буквы, цифры, подчёркивания
import re as _re
_SAFE_NAME = _re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
for col_name, col_type in stage2_columns.items(): for col_name, col_type in stage2_columns.items():
if not _SAFE_NAME.match(col_name):
logger.error(f"Unsafe column name skipped: {col_name}")
continue
if not _SAFE_NAME.match(col_type.split()[0]):
logger.error(f"Unsafe column type skipped: {col_type}")
continue
col_exists = await conn.fetchval(""" col_exists = await conn.fetchval("""
SELECT 1 FROM information_schema.columns SELECT 1 FROM information_schema.columns
WHERE table_name = 'user_skill_mastery' AND column_name = $1 WHERE table_name = 'user_skill_mastery' AND column_name = $1
@@ -904,28 +944,32 @@ async def init_db():
""") """)
# ============ Индексы ============ # ============ Индексы ============
# 🔧 ФИКС (24.09.2026): redundant_index_users_telegram —
# убраны дубли на UNIQUE-колонках:
# - idx_users_telegram (telegram_id уже UNIQUE)
# - idx_typing_progress_user (user_id+sentence_id уже PK)
# - idx_user_achievements_user (user_id+achievement_id уже PK)
# - idx_daily_logins_user (user_id+login_date уже PK)
# - idx_skill_mastery_user (user_id+skill_tag уже UNIQUE)
# - idx_user_progress_user (user_id+level_id уже PK)
# - idx_sentence_progress_user (user_id+sentence_id уже PK)
await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_number ON levels(level_number)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_number ON levels(level_number)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_level ON levels(level)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_level ON levels(level)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_level_number ON levels(level, level_number)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_level_number ON levels(level, level_number)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_user_progress_user ON user_progress(user_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_level_words_level_id ON level_words(level_id)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_level_words_level_id ON level_words(level_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_level_words_level ON level_words(level)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_level_words_level ON level_words(level)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_users_telegram ON users(telegram_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_number ON sentences(sentence_number)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_number ON sentences(sentence_number)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_level ON sentences(level)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_level ON sentences(level)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_level_number ON sentences(level, sentence_number)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_level_number ON sentences(level, sentence_number)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentence_progress_user ON sentence_progress(user_id)") # Оставляем score-индексы (не дубли)
await conn.execute("CREATE INDEX IF NOT EXISTS idx_typing_progress_user ON typing_progress(user_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_user_progress_score ON user_progress(score DESC)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_user_progress_score ON user_progress(score DESC)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentence_progress_score ON sentence_progress(score DESC)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentence_progress_score ON sentence_progress(score DESC)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_user_achievements_user ON user_achievements(user_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_daily_logins_user ON daily_logins(user_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_daily_logins_date ON daily_logins(login_date DESC)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_daily_logins_date ON daily_logins(login_date DESC)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_topics_level ON learning_topics(level_code)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_topics_level ON learning_topics(level_code)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_lessons_topic ON learning_lessons(topic_id)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_lessons_topic ON learning_lessons(topic_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_questions_lesson ON learning_questions(lesson_id)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_questions_lesson ON learning_questions(lesson_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_questions_topic ON learning_questions(topic_id)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_questions_topic ON learning_questions(topic_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_answers_user ON user_learning_answers(user_id, created_at DESC)") await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_answers_user ON user_learning_answers(user_id, created_at DESC)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_skill_mastery_user ON user_skill_mastery(user_id)") # конец транзакции
logger.info("Database initialized") logger.info("Database initialized")
+44 -8
View File
@@ -1,4 +1,5 @@
import asyncio import asyncio
import ipaddress
import logging import logging
import os import os
import re import re
@@ -10,7 +11,7 @@ from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
from config import FRONTEND_ORIGIN, UPLOADS_DIR from config import FRONTEND_ORIGIN, UPLOADS_DIR, CF_IPS
from database import init_db from database import init_db
from routers import users, game, hints, leaderboard, sentences, achievements, wordquiz, bubblewords, pronunciation, assistant, learning, tts, typing, teacher, support, admin, feed, admin_content from routers import users, game, hints, leaderboard, sentences, achievements, wordquiz, bubblewords, pronunciation, assistant, learning, tts, typing, teacher, support, admin, feed, admin_content
from routers.sentences import init_sentences from routers.sentences import init_sentences
@@ -125,6 +126,25 @@ class RateLimiter:
rate_limiter = RateLimiter() rate_limiter = RateLimiter()
# 🔧 ФИКС (24.09.2026): cf_connecting_ip_spoof —
# проверяем, что IP клиента входит в список Cloudflare.
def _is_cloudflare_ip(ip_str: str) -> bool:
"""Проверяет, что IP входит в CF_IPS (Cloudflare)."""
if not ip_str:
return False
try:
ip = ipaddress.ip_address(ip_str)
except ValueError:
return False
for cidr in CF_IPS:
try:
if ip in ipaddress.ip_network(cidr, strict=False):
return True
except ValueError:
continue
return False
# ============ Фоновое обновление кэша рейтинга ============ # ============ Фоновое обновление кэша рейтинга ============
async def refresh_leaderboard_cache_loop(): async def refresh_leaderboard_cache_loop():
await asyncio.sleep(10) await asyncio.sleep(10)
@@ -151,7 +171,9 @@ async def lifespan(app: FastAPI):
await init_achievements() await init_achievements()
await init_learning_data() await init_learning_data()
asyncio.create_task(rate_limiter.start_cleanup()) # 🔧 ФИКС (24.09.2026): cleanup_task_not_cancelled —
# сохраняем ссылку и отменяем в shutdown.
cleanup_task = asyncio.create_task(rate_limiter.start_cleanup())
from database import get_pool from database import get_pool
pool = await get_pool() pool = await get_pool()
@@ -218,6 +240,14 @@ async def lifespan(app: FastAPI):
except asyncio.CancelledError: except asyncio.CancelledError:
logger.info("Leaderboard refresh task cancelled") logger.info("Leaderboard refresh task cancelled")
# 🔧 ФИКС (24.09.2026): cleanup_task_not_cancelled —
# отменяем cleanup-задачу rate limiter.
cleanup_task.cancel()
try:
await cleanup_task
except asyncio.CancelledError:
logger.info("Rate limiter cleanup task cancelled")
logger.info(f"Filwords API v{APP_VERSION} stopped") logger.info(f"Filwords API v{APP_VERSION} stopped")
@@ -232,16 +262,22 @@ async def rate_limit_middleware(request: Request, call_next):
if path in ["/", "/health"]: if path in ["/", "/health"]:
return await call_next(request) return await call_next(request)
# 🔧 ФИКС (23.09.2026): xforwarded_for_spoof — раньше любой клиент мог # 🔧 ФИКС (24.09.2026): cf_connecting_ip_spoof —
# подставить X-Forwarded-For и обойти rate limit. # доверяем cf-connecting-ip ТОЛЬКО если запрос пришёл с IP Cloudflare.
# Теперь порядок: cf-connecting-ip (Cloudflare) → x-forwarded-for (только если # Иначе — берём request.client.host (реальный IP клиента).
# запрос пришёл с доверенного прокси) → request.client.host. real_client_ip = request.client.host if request.client else "unknown"
# TODO: вынести CF_IPS в конфиг и проверять источник по списку.
if _is_cloudflare_ip(real_client_ip):
# Запрос от Cloudflare — доверяем cf-connecting-ip
client_ip = ( client_ip = (
request.headers.get("cf-connecting-ip") request.headers.get("cf-connecting-ip")
or request.headers.get("x-real-ip") or request.headers.get("x-real-ip")
or (request.client.host if request.client else "unknown") or real_client_ip
) )
else:
# Запрос не от Cloudflare — игнорируем заголовки, берём реальный IP
client_ip = real_client_ip
rate_key = f"{client_ip}:{path}" rate_key = f"{client_ip}:{path}"
if not rate_limiter.is_allowed(rate_key, path): if not rate_limiter.is_allowed(rate_key, path):
+59 -1
View File
@@ -424,7 +424,8 @@ async def admin_update_feed_post(request: Request, post_id: int):
"is_pinned": true/false, "is_pinned": true/false,
"is_published": true/false, "is_published": true/false,
"expires_at": "..." | null, "expires_at": "..." | null,
"translate_again": true — перевести заново через AI "translate_again": true, — перевести заново через AI
"removed_images": ["..."] — 🔧 ФИКС (24.09.2026): URL'ы удалённых картинок
} }
""" """
telegram_id, _ = await require_auth(request) telegram_id, _ = await require_auth(request)
@@ -443,6 +444,38 @@ async def admin_update_feed_post(request: Request, post_id: int):
source_language = existing["source_language"] source_language = existing["source_language"]
# 🔧 ФИКС (24.09.2026): existing_image_removal_no_delete —
# обрабатываем удалённые картинки: удаляем из БД и с диска.
removed_images = body.get("removed_images", [])
if removed_images and isinstance(removed_images, list):
for img_url in removed_images:
# img_url приходит как полный URL: https://api.filwordspl.com/uploads/feed/...
# или относительный: /uploads/feed/...
# Приводим к относительному виду для БД
rel_path = img_url
if img_url.startswith("http"):
# Вырезаем /uploads/...
idx = img_url.find("/uploads/")
if idx >= 0:
rel_path = img_url[idx:]
else:
logger.warning(f"[removed_images] cannot parse URL: {img_url}")
continue
# Удаляем из БД
deleted = await conn.execute("""
DELETE FROM feed_attachments
WHERE post_id = $1 AND file_path = $2
""", post_id, rel_path)
deleted_count = int(deleted.split()[-1]) if deleted else 0
if deleted_count > 0:
logger.info(f"[removed_images] deleted from DB: {rel_path}")
else:
logger.warning(f"[removed_images] not found in DB: {rel_path}")
logger.info(f"[removed_images] post #{post_id}: processed {len(removed_images)} items")
# Мерджим поля: если не передано — оставляем старое # Мерджим поля: если не передано — оставляем старое
title = body.get("title", existing.get(f"title_{source_language}") or "") title = body.get("title", existing.get(f"title_{source_language}") or "")
post_body = body.get("body", existing.get(f"body_{source_language}") or "") post_body = body.get("body", existing.get(f"body_{source_language}") or "")
@@ -516,6 +549,31 @@ async def admin_update_feed_post(request: Request, post_id: int):
post_id, post_id,
) )
# 🔧 ФИКС (24.09.2026): existing_image_removal_no_delete —
# удаляем файлы с диска ВНЕ pool.acquire.
if removed_images and isinstance(removed_images, list):
for img_url in removed_images:
rel_path = img_url
if img_url.startswith("http"):
idx = img_url.find("/uploads/")
if idx >= 0:
rel_path = img_url[idx:]
else:
continue
# "/uploads/feed/16_0_abc.jpg" → "feed/16_0_abc.jpg"
rel_disk = rel_path.replace("/uploads/", "", 1).lstrip("/")
abs_path = Path(UPLOADS_DIR) / rel_disk
try:
if abs_path.exists():
abs_path.unlink()
logger.info(f"[removed_images] removed file: {abs_path}")
else:
logger.warning(f"[removed_images] file not found: {abs_path}")
except Exception as e:
logger.error(f"[removed_images] failed to remove {abs_path}: {e}")
return {"status": "ok", "post_id": post_id} return {"status": "ok", "post_id": post_id}