🌊 Волна 14: backend #6 — security + БД (8 багов + бонус)

Security:
- cf_connecting_ip_spoof: whitelist CF_IPS, доверие только от Cloudflare (config.py + main.py)

БД:
- get_pool_race: asyncio.Lock вокруг create_pool (database.py)
- init_db_no_transaction: обёрнут в conn.transaction() (database.py)
- drop_constraint_cascade: убран CASCADE (database.py)
- add_pk_on_duplicates: DELETE дублей перед ADD PK (database.py)
- redundant_index_users_telegram: удалены дубли индексов (database.py)
- fstring_sql_migration: whitelist regex (database.py)

Качество:
- cleanup_task_not_cancelled: cancel в shutdown (main.py)

Бонус:
- removed_images: обработка удалённых картинок в feed.py (TODO из v1)

Пропущено осознанно:
- matview_if_not_exists_stale (Alembic, отдельно)
- initdata_in_query_param (fallback для deleteFeedPost)

Аудит v3: Issue #6, backend 8/10 задач
This commit is contained in:
2026-09-24 15:43:13 +00:00
parent 742337fb14
commit 4152b4b2c4
4 changed files with 1024 additions and 871 deletions
+15
View File
@@ -35,6 +35,21 @@ MAX_INITDATA_AGE = 4 * 60 * 60 # 4 часа в секундах
SUPPORTED_LANGUAGES = {"pl", "ru", "uk", "en"}
# 🔧 ФИКС (24.09.2026): cf_connecting_ip_spoof —
# доверяем заголовку cf-connecting-ip ТОЛЬКО если запрос пришёл
# с IP Cloudflare. Иначе — берём request.client.host.
# Список: https://www.cloudflare.com/ips-v4
CF_IPS = {
# IPv4
"173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22", "103.31.4.0/22",
"141.101.64.0/18", "108.162.192.0/18", "190.93.240.0/20", "188.114.96.0/20",
"197.234.240.0/22", "198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13",
"104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22",
# IPv6
"2400:cb00::/32", "2606:4700::/32", "2803:f800::/32",
"2405:b500::/32", "2405:8100::/32", "2a06:98c0::/29", "2c0f:f248::/32",
}
# ============ 🔧 НОВОЕ (18.09.2026): настройки для FeedModal ============
# Папка для хранения картинок feed-постов.
# Путь совпадает с backend/, файлы доступны как static mounts (см. main.py при необходимости).
+52 -8
View File
@@ -1,12 +1,19 @@
import asyncio
import asyncpg
import logging
from config import DB_CONFIG
logger = logging.getLogger(__name__)
db_pool = None
# 🔧 ФИКС (24.09.2026): get_pool_race — защита от гонки при первом обращении.
_pool_lock = asyncio.Lock()
async def get_pool():
global db_pool
if db_pool is not None:
return db_pool
# 🔧 Двойная проверка: сначала без лока, потом внутри лока
async with _pool_lock:
if db_pool is None:
db_pool = await asyncpg.create_pool(**DB_CONFIG, min_size=2, max_size=10)
return db_pool
@@ -14,6 +21,10 @@ async def get_pool():
async def init_db():
pool = await get_pool()
async with pool.acquire() as conn:
# 🔧 ФИКС (24.09.2026): init_db_no_transaction —
# оборачиваем ВСЮ инициализацию в одну транзакцию.
# Если упадёт на середине — БД вернётся в исходное состояние.
async with conn.transaction():
# ============ users ============
await conn.execute("""
CREATE TABLE IF NOT EXISTS users (
@@ -104,9 +115,12 @@ async def init_db():
if existing and existing['data_type'] != 'bigint':
logger.info("Migrating user_progress.user_id from INTEGER to BIGINT")
# 🔧 ФИКС (24.09.2026): drop_constraint_cascade —
# убран CASCADE, чтобы не снести FK из других таблиц.
# Перечисляем зависимые FK явно (пересоздадим ниже).
await conn.execute("""
ALTER TABLE user_progress
DROP CONSTRAINT IF EXISTS user_progress_pkey CASCADE
DROP CONSTRAINT IF EXISTS user_progress_pkey
""")
await conn.execute("""
ALTER TABLE user_progress
@@ -116,6 +130,18 @@ async def init_db():
ALTER TABLE user_progress
ALTER COLUMN user_id TYPE BIGINT
""")
# 🔧 ФИКС (24.09.2026): add_pk_on_duplicates —
# удаляем дубли перед ADD PRIMARY KEY.
# Оставляем строку с максимальным completed_at (или первую).
await conn.execute("""
DELETE FROM user_progress a
USING user_progress b
WHERE a.ctid < b.ctid
AND a.user_id = b.user_id
AND a.level_id = b.level_id
""")
await conn.execute("""
ALTER TABLE user_progress
ADD PRIMARY KEY (user_id, level_id)
@@ -506,6 +532,9 @@ async def init_db():
""")
# ============ Авто-миграции для user_skill_mastery (Этап 2) ============
# 🔧 ФИКС (24.09.2026): fstring_sql_migration —
# whitelist-проверка имён колонок и типов. Хардкод безопасен,
# но если кто-то расширит словарь — может быть SQL-инъекция.
stage2_columns = {
"previous_mastery_score": "NUMERIC(5,2) DEFAULT 0",
"mastery_delta": "NUMERIC(5,2) DEFAULT 0",
@@ -516,7 +545,18 @@ async def init_db():
"active_knowledge_score": "NUMERIC(5,2) DEFAULT 0",
}
# Whitelist: только буквы, цифры, подчёркивания
import re as _re
_SAFE_NAME = _re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
for col_name, col_type in stage2_columns.items():
if not _SAFE_NAME.match(col_name):
logger.error(f"Unsafe column name skipped: {col_name}")
continue
if not _SAFE_NAME.match(col_type.split()[0]):
logger.error(f"Unsafe column type skipped: {col_type}")
continue
col_exists = await conn.fetchval("""
SELECT 1 FROM information_schema.columns
WHERE table_name = 'user_skill_mastery' AND column_name = $1
@@ -904,28 +944,32 @@ async def init_db():
""")
# ============ Индексы ============
# 🔧 ФИКС (24.09.2026): redundant_index_users_telegram —
# убраны дубли на UNIQUE-колонках:
# - idx_users_telegram (telegram_id уже UNIQUE)
# - idx_typing_progress_user (user_id+sentence_id уже PK)
# - idx_user_achievements_user (user_id+achievement_id уже PK)
# - idx_daily_logins_user (user_id+login_date уже PK)
# - idx_skill_mastery_user (user_id+skill_tag уже UNIQUE)
# - idx_user_progress_user (user_id+level_id уже PK)
# - idx_sentence_progress_user (user_id+sentence_id уже PK)
await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_number ON levels(level_number)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_level ON levels(level)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_levels_level_number ON levels(level, level_number)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_user_progress_user ON user_progress(user_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_level_words_level_id ON level_words(level_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_level_words_level ON level_words(level)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_users_telegram ON users(telegram_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_number ON sentences(sentence_number)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_level ON sentences(level)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentences_level_number ON sentences(level, sentence_number)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentence_progress_user ON sentence_progress(user_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_typing_progress_user ON typing_progress(user_id)")
# Оставляем score-индексы (не дубли)
await conn.execute("CREATE INDEX IF NOT EXISTS idx_user_progress_score ON user_progress(score DESC)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_sentence_progress_score ON sentence_progress(score DESC)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_user_achievements_user ON user_achievements(user_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_daily_logins_user ON daily_logins(user_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_daily_logins_date ON daily_logins(login_date DESC)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_topics_level ON learning_topics(level_code)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_lessons_topic ON learning_lessons(topic_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_questions_lesson ON learning_questions(lesson_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_questions_topic ON learning_questions(topic_id)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_learning_answers_user ON user_learning_answers(user_id, created_at DESC)")
await conn.execute("CREATE INDEX IF NOT EXISTS idx_skill_mastery_user ON user_skill_mastery(user_id)")
# конец транзакции
logger.info("Database initialized")
+44 -8
View File
@@ -1,4 +1,5 @@
import asyncio
import ipaddress
import logging
import os
import re
@@ -10,7 +11,7 @@ from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from config import FRONTEND_ORIGIN, UPLOADS_DIR
from config import FRONTEND_ORIGIN, UPLOADS_DIR, CF_IPS
from database import init_db
from routers import users, game, hints, leaderboard, sentences, achievements, wordquiz, bubblewords, pronunciation, assistant, learning, tts, typing, teacher, support, admin, feed, admin_content
from routers.sentences import init_sentences
@@ -125,6 +126,25 @@ class RateLimiter:
rate_limiter = RateLimiter()
# 🔧 ФИКС (24.09.2026): cf_connecting_ip_spoof —
# проверяем, что IP клиента входит в список Cloudflare.
def _is_cloudflare_ip(ip_str: str) -> bool:
"""Проверяет, что IP входит в CF_IPS (Cloudflare)."""
if not ip_str:
return False
try:
ip = ipaddress.ip_address(ip_str)
except ValueError:
return False
for cidr in CF_IPS:
try:
if ip in ipaddress.ip_network(cidr, strict=False):
return True
except ValueError:
continue
return False
# ============ Фоновое обновление кэша рейтинга ============
async def refresh_leaderboard_cache_loop():
await asyncio.sleep(10)
@@ -151,7 +171,9 @@ async def lifespan(app: FastAPI):
await init_achievements()
await init_learning_data()
asyncio.create_task(rate_limiter.start_cleanup())
# 🔧 ФИКС (24.09.2026): cleanup_task_not_cancelled —
# сохраняем ссылку и отменяем в shutdown.
cleanup_task = asyncio.create_task(rate_limiter.start_cleanup())
from database import get_pool
pool = await get_pool()
@@ -218,6 +240,14 @@ async def lifespan(app: FastAPI):
except asyncio.CancelledError:
logger.info("Leaderboard refresh task cancelled")
# 🔧 ФИКС (24.09.2026): cleanup_task_not_cancelled —
# отменяем cleanup-задачу rate limiter.
cleanup_task.cancel()
try:
await cleanup_task
except asyncio.CancelledError:
logger.info("Rate limiter cleanup task cancelled")
logger.info(f"Filwords API v{APP_VERSION} stopped")
@@ -232,16 +262,22 @@ async def rate_limit_middleware(request: Request, call_next):
if path in ["/", "/health"]:
return await call_next(request)
# 🔧 ФИКС (23.09.2026): xforwarded_for_spoof — раньше любой клиент мог
# подставить X-Forwarded-For и обойти rate limit.
# Теперь порядок: cf-connecting-ip (Cloudflare) → x-forwarded-for (только если
# запрос пришёл с доверенного прокси) → request.client.host.
# TODO: вынести CF_IPS в конфиг и проверять источник по списку.
# 🔧 ФИКС (24.09.2026): cf_connecting_ip_spoof —
# доверяем cf-connecting-ip ТОЛЬКО если запрос пришёл с IP Cloudflare.
# Иначе — берём request.client.host (реальный IP клиента).
real_client_ip = request.client.host if request.client else "unknown"
if _is_cloudflare_ip(real_client_ip):
# Запрос от Cloudflare — доверяем cf-connecting-ip
client_ip = (
request.headers.get("cf-connecting-ip")
or request.headers.get("x-real-ip")
or (request.client.host if request.client else "unknown")
or real_client_ip
)
else:
# Запрос не от Cloudflare — игнорируем заголовки, берём реальный IP
client_ip = real_client_ip
rate_key = f"{client_ip}:{path}"
if not rate_limiter.is_allowed(rate_key, path):
+59 -1
View File
@@ -424,7 +424,8 @@ async def admin_update_feed_post(request: Request, post_id: int):
"is_pinned": true/false,
"is_published": true/false,
"expires_at": "..." | null,
"translate_again": true — перевести заново через AI
"translate_again": true, — перевести заново через AI
"removed_images": ["..."] — 🔧 ФИКС (24.09.2026): URL'ы удалённых картинок
}
"""
telegram_id, _ = await require_auth(request)
@@ -443,6 +444,38 @@ async def admin_update_feed_post(request: Request, post_id: int):
source_language = existing["source_language"]
# 🔧 ФИКС (24.09.2026): existing_image_removal_no_delete —
# обрабатываем удалённые картинки: удаляем из БД и с диска.
removed_images = body.get("removed_images", [])
if removed_images and isinstance(removed_images, list):
for img_url in removed_images:
# img_url приходит как полный URL: https://api.filwordspl.com/uploads/feed/...
# или относительный: /uploads/feed/...
# Приводим к относительному виду для БД
rel_path = img_url
if img_url.startswith("http"):
# Вырезаем /uploads/...
idx = img_url.find("/uploads/")
if idx >= 0:
rel_path = img_url[idx:]
else:
logger.warning(f"[removed_images] cannot parse URL: {img_url}")
continue
# Удаляем из БД
deleted = await conn.execute("""
DELETE FROM feed_attachments
WHERE post_id = $1 AND file_path = $2
""", post_id, rel_path)
deleted_count = int(deleted.split()[-1]) if deleted else 0
if deleted_count > 0:
logger.info(f"[removed_images] deleted from DB: {rel_path}")
else:
logger.warning(f"[removed_images] not found in DB: {rel_path}")
logger.info(f"[removed_images] post #{post_id}: processed {len(removed_images)} items")
# Мерджим поля: если не передано — оставляем старое
title = body.get("title", existing.get(f"title_{source_language}") or "")
post_body = body.get("body", existing.get(f"body_{source_language}") or "")
@@ -516,6 +549,31 @@ async def admin_update_feed_post(request: Request, post_id: int):
post_id,
)
# 🔧 ФИКС (24.09.2026): existing_image_removal_no_delete —
# удаляем файлы с диска ВНЕ pool.acquire.
if removed_images and isinstance(removed_images, list):
for img_url in removed_images:
rel_path = img_url
if img_url.startswith("http"):
idx = img_url.find("/uploads/")
if idx >= 0:
rel_path = img_url[idx:]
else:
continue
# "/uploads/feed/16_0_abc.jpg" → "feed/16_0_abc.jpg"
rel_disk = rel_path.replace("/uploads/", "", 1).lstrip("/")
abs_path = Path(UPLOADS_DIR) / rel_disk
try:
if abs_path.exists():
abs_path.unlink()
logger.info(f"[removed_images] removed file: {abs_path}")
else:
logger.warning(f"[removed_images] file not found: {abs_path}")
except Exception as e:
logger.error(f"[removed_images] failed to remove {abs_path}: {e}")
return {"status": "ok", "post_id": post_id}