Personal/manga
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

upd

Browse Source
This commit is contained in:
StenFredd
2026-04-30 18:54:24 +03:00
parent 88bf301b60
commit 87b692ba49
8 changed files with 1545 additions and 691 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
src/api.py
View File

@@ -20,6 +20,7 @@ from loguru import logger
from .state import StateDB
from .worker import download_manga, check_for_updates
from .exporter import patch_meta, MangaMeta
from .sources import registry, get_source_for_url, extract_domain
OUTPUT_DIR = Path("/app/output")
FRONTEND_DIR = Path("/app/frontend")
@@ -172,6 +173,16 @@ async def _queue_worker_loop():
@app.on_event("startup")
async def startup_event():
# Синхронизируем источники с кодом и мигрируем существующие манги
_db = StateDB()
try:
_db.sync_sources(registry)
migrated = _db.migrate_manga_sources()
if migrated:
logger.info("Авто-миграция: проставлен source_id для {} манг", migrated)
finally:
_db.close()
asyncio.create_task(queue_worker())
asyncio.create_task(update_scheduler())
# Восстанавливаем очередь из БД (незавершённые задачи)
@@ -365,6 +376,16 @@ def _enrich_manga(m: dict, db: StateDB) -> dict:
AND pages_total > 0 AND pages_done < pages_total""",
(m["url"],)
).fetchone()[0]
# Источник
source_info = None
if m.get("source_id"):
src = db.get_source_by_id(m["source_id"])
if src:
source_info = {"id": src["id"], "slug": src["slug"], "display_name": src["display_name"]}
else:
source_info = {"id": m["source_id"], "slug": "unknown", "display_name": "Источник недоступен"}
return {
**m,
"chapters_done": ch_done_count,
@@ -375,6 +396,7 @@ def _enrich_manga(m: dict, db: StateDB) -> dict:
"errors_count": ch_failed + ch_partial,
"started_at": m.get("started_at"),
"finished_at": m.get("finished_at"),
"source": source_info,
}
@@ -454,6 +476,7 @@ def _manga_detail(manga: dict, db: StateDB) -> dict:
class AddMangaRequest(BaseModel):
urls: List[str]
format: str = "cbz"
source_id: Optional[int] = None # явный выбор источника (для неизвестных доменов)
# ── Auth API ─────────────────────────────────
@@ -536,7 +559,24 @@ async def add_to_queue(body: AddMangaRequest):
url = url.strip()
if not url:
continue
is_new = db.add_manga(url, body.format)
# Определяем source_id: явный из запроса или авто по домену
source_id = body.source_id
if source_id is None:
domain = extract_domain(url)
source_row = db.get_source_by_domain(domain)
if source_row:
source_id = source_row["id"]
# Если источник указан явно — привязываем домен к нему
if body.source_id is not None:
domain = extract_domain(url)
existing = db.get_source_by_domain(domain)
if existing and existing["id"] != body.source_id:
db.remove_domain(existing["id"], domain)
db.add_domain(body.source_id, domain)
is_new = db.add_manga(url, body.format, source_id=source_id)
if is_new:
await download_queue.put({"url": url, "fmt": body.format})
added.append(url)
@@ -544,9 +584,9 @@ async def add_to_queue(body: AddMangaRequest):
"type": "manga_queued",
"url": url,
"format": body.format,
"source_id": source_id,
})
await _broadcast_queue_positions()
# Запускаем фоновую задачу предпросмотра (без Chromium — быстро)
asyncio.create_task(_fetch_preview(url))
else:
skipped.append(url)
@@ -559,15 +599,27 @@ async def _fetch_preview(url: str):
"""Быстро получает название и количество глав сразу после добавления."""
try:
from .browser import BrowserManager
from .scraper import get_manga_info
async with BrowserManager(headless=True) as bm:
_, page = await bm.new_page()
manga = await get_manga_info(page, url)
if not manga:
return
db = StateDB()
try:
db.update_manga_info(
source = get_source_for_url(url, db)
if source is None:
manga_row = db.get_manga(url)
if manga_row and manga_row.get("source_id"):
source = registry.get_by_db_id(manga_row["source_id"], db)
finally:
db.close()
if source is None:
return
async with BrowserManager(headless=True) as bm:
_, page = await bm.new_page()
manga = await source.get_manga_info(page, url)
if not manga:
return
db2 = StateDB()
try:
db2.update_manga_info(
url,
title=manga.title_ru or manga.title,
chapters_total=len(manga.chapters),
@@ -576,7 +628,7 @@ async def _fetch_preview(url: str):
pub_status=manga.pub_status,
)
finally:
db.close()
db2.close()
await ws_manager.broadcast({
"type": "manga_preview",
"url": url,
@@ -996,6 +1048,151 @@ async def delete_manga(url: str, delete_files: bool = False):
db.close()
# ── Sources API ───────────────────────────────
class DomainAdd(BaseModel):
domain: str
class SwitchSourceRequest(BaseModel):
url: str
source_id: int
@app.get("/api/sources")
async def list_sources():
"""Список всех источников с доменами."""
db = StateDB()
try:
return db.get_all_sources()
finally:
db.close()
@app.get("/api/resolve-source")
async def resolve_source(url: str):
"""Определить источник по URL. Возвращает {id, slug, display_name} или null."""
db = StateDB()
try:
domain = extract_domain(url)
row = db.get_source_by_domain(domain)
if not row:
return {"source": None, "domain": domain}
return {
"source": {
"id": row["id"],
"slug": row["slug"],
"display_name": row["display_name"],
},
"domain": domain,
}
finally:
db.close()
@app.post("/api/sources/{source_id}/domains")
async def add_domain(source_id: int, body: DomainAdd):
"""Добавить домен к источнику."""
db = StateDB()
try:
source = db.get_source_by_id(source_id)
if not source:
raise HTTPException(status_code=404, detail="Источник не найден")
domain = body.domain.lower().strip()
if not domain:
raise HTTPException(status_code=400, detail="Домен не может быть пустым")
# Проверяем не занят ли домен другим источником
existing = db.get_source_by_domain(domain)
if existing and existing["id"] != source_id:
raise HTTPException(
status_code=409,
detail=f"Домен уже привязан к источнику «{existing['display_name']}»"
)
ok = db.add_domain(source_id, domain)
if not ok:
raise HTTPException(status_code=409, detail="Домен уже существует")
await ws_manager.broadcast({
"type": "source_domain_added",
"source_id": source_id,
"domain": domain,
})
return {"ok": True, "domain": domain}
finally:
db.close()
@app.delete("/api/sources/{source_id}/domains/{domain:path}")
async def remove_domain(source_id: int, domain: str):
"""Удалить домен у источника."""
db = StateDB()
try:
source = db.get_source_by_id(source_id)
if not source:
raise HTTPException(status_code=404, detail="Источник не найден")
ok = db.remove_domain(source_id, domain)
if not ok:
raise HTTPException(status_code=404, detail="Домен не найден")
await ws_manager.broadcast({
"type": "source_domain_removed",
"source_id": source_id,
"domain": domain,
})
return {"ok": True}
finally:
db.close()
@app.post("/api/mangas/switch-source")
async def switch_manga_source(body: SwitchSourceRequest):
"""Сменить источник у манги + перепривязать домен."""
db = StateDB()
try:
manga = db.get_manga(body.url)
if not manga:
raise HTTPException(status_code=404, detail="Манга не найдена")
if manga["status"] == "downloading" and body.url in active_tasks:
raise HTTPException(status_code=400, detail="Нельзя сменить источник во время загрузки")
new_source = db.get_source_by_id(body.source_id)
if not new_source:
raise HTTPException(status_code=404, detail="Источник не найден")
old_source_id = manga.get("source_id")
domain = extract_domain(body.url)
# Перепривязываем домен
if domain:
existing_domain = db.get_source_by_domain(domain)
if existing_domain and existing_domain["id"] != body.source_id:
db.remove_domain(existing_domain["id"], domain)
db.add_domain(body.source_id, domain)
# Меняем источник у манги
db.set_manga_source(body.url, body.source_id)
# Сбрасываем failed/partial главы → pending
reset_count = db.reset_failed_chapters(body.url)
await ws_manager.broadcast({
"type": "source_switched",
"url": body.url,
"old_source_id": old_source_id,
"new_source_id": body.source_id,
"new_source_name": new_source["display_name"],
"domain_rebound": bool(domain),
"chapters_reset": reset_count,
})
return {
"ok": True,
"source_id": body.source_id,
"source_name": new_source["display_name"],
"chapters_reset": reset_count,
}
finally:
db.close()
@app.get("/api/stats")
async def global_stats():
db = StateDB()

668
src/scraper.py
View File

@@ -1,665 +1,19 @@
"""
Парсер readmanga.ru: список глав и URL/байты изображений внутри главы.
Обратно-совместимый shim: делегирует вызовы ReadmangaSource.
Не используйте напрямую в новом коде — используйте src.sources.registry.
"""
import asyncio
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
from .sources.base import Chapter, MangaInfo # noqa: F401 — реэкспорт для импортёров
from .sources.readmanga import ReadmangaSource
from loguru import logger
from playwright.async_api import Page
from .browser import BrowserManager
_instance = ReadmangaSource()
# ──────────────────────────────────────────────
# Модели данных
# ──────────────────────────────────────────────
@dataclass
class Chapter:
title: str
url: str
number: float = 0.0
volume: int = 0
async def get_manga_info(page, url):
return await _instance.get_manga_info(page, url)
@dataclass
class MangaInfo:
title: str
url: str
chapters: list[Chapter] = field(default_factory=list)
pub_status: str = "unknown" # completed / ongoing / unknown
title_ru: str = "" # Только русский тайтл (для папки)
title_full: str = "" # Полный тайтл как на странице
description: str = "" # Описание/синопсис
genres: list[str] = field(default_factory=list) # Жанры
# ──────────────────────────────────────────────
# Страница манги — список глав
# ──────────────────────────────────────────────
async def get_manga_info(page: Page, url: str) -> Optional[MangaInfo]:
"""Открывает страницу манги и возвращает список всех глав."""
logger.info("Загружаем страницу манги: {}", url)
ok = await _navigate(page, url)
if not ok:
return None
title_full = await page.title()
title_full = re.sub(r"\s*[-–|].*$", "", title_full).strip()
# Пробуем взять русский тайтл напрямую из DOM
title_ru = await _extract_ru_title_from_dom(page)
if not title_ru:
title_ru = _parse_ru_title(title_full)
logger.info("Манга: {} | ru: {}", title_full, title_ru)
pub_status = await _extract_pub_status(page)
logger.info("Статус выпуска: {}", pub_status)
description = await _extract_description(page)
genres = await _extract_genres(page)
await _expand_chapters(page)
chapters = await _extract_chapters(page)
if not chapters:
chapters = await _extract_chapters_alt(page)
logger.info("Найдено глав: {}", len(chapters))
return MangaInfo(
title=title_ru or title_full,
url=url,
chapters=chapters,
pub_status=pub_status,
title_ru=title_ru,
title_full=title_full,
description=description,
genres=genres,
async def get_chapter_images_and_download(page, chapter_url, dest_dir,
manga_url=None, on_page=None):
return await _instance.get_chapter_images_and_download(
page, chapter_url, dest_dir, manga_url=manga_url, on_page=on_page
)
async def _extract_ru_title_from_dom(page: Page) -> str:
"""Ищет русский тайтл в структуре страницы readmanga."""
try:
result = await page.evaluate("""
() => {
// readmanga: основной тайтл в span.name внутри .names
const selectors = [
'.names .name',
'h1.manga-title',
'h1 .name',
'.name-block .name',
];
for (const sel of selectors) {
const el = document.querySelector(sel);
if (el && el.textContent.trim()) return el.textContent.trim();
}
return '';
}
""")
return (result or "").strip()
except Exception:
return ""
def _parse_ru_title(full_title: str) -> str:
"""Извлекает русский тайтл из полной строки тайтла.
Примеры:
'Манга Режим — АД. Хардкорный геймер ... (Hellmode)' → 'Режим — АД. Хардкорный геймер ...'
'Манга Магическая битва (Sorcery Fight) Гэгэ онлайн' → 'Магическая битва'
'Авантюрист Monster Eater Adventurer' → 'Авантюрист'
"""
t = full_title.strip()
# Убираем префикс "Манга "
t = re.sub(r'^Манга\s+', '', t).strip()
# Берём только до первой скобки (начало английского тайтла)
t = re.split(r'\s*[\(\[]', t)[0].strip()
# Убираем суффикс " онлайн"
t = re.sub(r'\s+онлайн\s*$', '', t, flags=re.IGNORECASE).strip()
# Обрезаем хвост из латинских слов.
# Правило: стоп только на токене содержащем латиницу (a-zA-Z).
# Пунктуация между кириллическими словами (—, –, ., :, !) — сохраняем.
words = t.split()
result = []
for w in words:
if re.search(r'[а-яёА-ЯЁ]', w):
result.append(w)
elif re.search(r'[a-zA-Z]', w):
# Первое латинское слово после кириллических — обрезаем здесь
if result:
break
else:
# Чисто пунктуационный токен (—, –, ., :, …)
# Добавляем только если уже есть кириллические слова (связка внутри)
if result:
result.append(w)
# Убираем висячую пунктуацию в конце (если последнее слово — не кириллица)
while result and not re.search(r'[а-яёА-ЯЁ]', result[-1]):
result.pop()
if result:
t = ' '.join(result)
return t
async def _extract_pub_status(page: Page) -> str:
"""Извлекает статус выпуска: completed / ongoing / unknown."""
try:
result = await page.evaluate("""
() => {
// readmanga хранит статус в .elem_status .value или похожих блоках
const statusSelectors = [
'.elem_status .value',
'.manga-info .status',
'[class*="status"] .value',
'.property .status',
];
for (const sel of statusSelectors) {
const el = document.querySelector(sel);
if (el) {
const t = el.textContent.toLowerCase();
if (t.includes('завершён') || t.includes('завершен') || t.includes('complete')) return 'completed';
if (t.includes('продолжает') || t.includes('ongoing')) return 'ongoing';
}
}
// Fallback: сканируем весь текст страницы
const bodyText = document.body ? document.body.innerText.toLowerCase() : '';
if (bodyText.includes('выпуск завершён') || bodyText.includes('выпуск завершен')) return 'completed';
if (bodyText.includes('продолжается')) return 'ongoing';
return 'unknown';
}
""")
return result or "unknown"
except Exception:
return "unknown"
async def _extract_description(page: Page) -> str:
"""Извлекает описание/синопсис манги."""
try:
result = await page.evaluate("""
() => {
const selectors = [
'.manga-description',
'.elem_descr .value',
'#tab-description .description-text',
'.description',
'[itemprop="description"]',
];
for (const sel of selectors) {
const el = document.querySelector(sel);
if (el && el.textContent.trim()) return el.textContent.trim();
}
return '';
}
""")
return (result or "").strip()[:2000] # обрезаем до 2000 символов
except Exception:
return ""
async def _extract_genres(page: Page) -> list[str]:
"""Извлекает список жанров манги."""
try:
result = await page.evaluate("""
() => {
const selectors = [
'.elem_genre .value a',
'.genres a',
'[itemprop="genre"]',
'.genre-list a',
];
for (const sel of selectors) {
const els = document.querySelectorAll(sel);
if (els.length) return Array.from(els).map(e => e.textContent.trim()).filter(Boolean);
}
return [];
}
""")
return result or []
except Exception:
return []
async def _navigate(page: Page, url: str, retries: int = 3,
referer: str | None = None) -> bool:
from urllib.parse import urlparse
if referer is None:
p = urlparse(url)
referer = f"{p.scheme}://{p.netloc}/"
for attempt in range(1, retries + 1):
try:
resp = await page.goto(url, wait_until="domcontentloaded",
timeout=60_000, referer=referer)
if resp and resp.status >= 400:
logger.warning("Попытка {}/{}: HTTP {}", attempt, retries, resp.status)
await asyncio.sleep(3 * attempt)
continue
try:
await page.wait_for_load_state("networkidle", timeout=10_000)
except Exception:
pass
return True
except Exception as e:
logger.warning("Попытка {}/{}: {}", attempt, retries, e)
await asyncio.sleep(3 * attempt)
return False
async def _expand_chapters(page: Page):
for sel in ["a.chapter-link.all", "button:has-text('Все главы')",
"a:has-text('Все главы')"]:
try:
el = page.locator(sel).first
if await el.is_visible(timeout=2000):
await el.click()
await page.wait_for_load_state("networkidle", timeout=10_000)
return
except Exception:
pass
async def _extract_chapters(page: Page) -> list[Chapter]:
"""Основной парсер: #chapters-list → tr.item-row → td[data-num] a.chapter-link"""
rows = await page.query_selector_all("#chapters-list tr.item-row")
chapters = []
for row in rows:
link = await row.query_selector("td[class*='item-title'] a")
if not link:
continue
href = await link.get_attribute("href") or ""
text = (await link.inner_text()).strip()
if not href:
continue
td = await row.query_selector("td[data-num]")
vol = int((await td.get_attribute("data-vol") or "0")) if td else 0
num_raw = int((await td.get_attribute("data-num") or "0")) if td else 0
number = num_raw / 10.0
full_url = href if href.startswith("http") else _base_url(page.url) + href
chapters.append(Chapter(title=text, url=full_url, number=number, volume=vol))
return chapters
async def _extract_chapters_alt(page: Page) -> list[Chapter]:
result = await page.evaluate("""
() => {
const links = Array.from(document.querySelectorAll('a[href*="/vol"]'));
return links.map(a => ({ href: a.href, text: a.textContent.trim() }))
.filter(x => x.href && x.text);
}
""")
return [Chapter(title=x["text"], url=x["href"],
number=_parse_num(x["text"]), volume=_parse_vol(x["text"]))
for x in result]
def _base_url(url: str) -> str:
m = re.match(r"(https?://[^/]+)", url)
return m.group(1) if m else "https://readmanga.ru"
def _parse_num(text: str) -> float:
m = re.search(r"[\d]+(?:[.,]\d+)?", text.replace(",", "."))
return float(m.group()) if m else 0.0
def _parse_vol(text: str) -> int:
m = re.search(r"Том\s+(\d+)", text, re.IGNORECASE)
return int(m.group(1)) if m else 0
# ──────────────────────────────────────────────
# Страница главы — получение URL изображений
# ──────────────────────────────────────────────
async def _extract_images_from_js(page: Page) -> list[str]:
"""
Извлекает URL из rm_h.readerInit(chapterInfo, [[base, '', path, w, h], ...]).
Считает скобки для точного захвата массива.
"""
try:
result = await page.evaluate("""
() => {
for (const s of document.querySelectorAll('script')) {
const text = s.textContent || '';
const mi = text.indexOf('readerInit');
if (mi === -1) continue;
const ai = text.indexOf('[', mi);
if (ai === -1) continue;
let depth = 0, end = -1;
for (let i = ai; i < text.length; i++) {
if (text[i] === '[') depth++;
else if (text[i] === ']') { depth--; if (!depth) { end = i+1; break; } }
}
if (end === -1) continue;
try {
const arr = eval(text.slice(ai, end));
if (Array.isArray(arr) && arr.length)
return arr.map(item => Array.isArray(item) && item.length >= 3
? item[0] + item[2] : null).filter(Boolean);
} catch(e) {}
}
return [];
}
""")
if result:
logger.debug("JS readerInit нашёл {} изображений", len(result))
return result or []
except Exception as e:
logger.debug("JS-метод не сработал: {}", e)
return []
async def _extract_images_from_dom(page: Page) -> list[str]:
try:
result = await page.evaluate("""
() => {
for (const sel of ['img.manga-page', '.page-image img', '#mangaReader img', 'img[data-src]']) {
const found = Array.from(document.querySelectorAll(sel));
if (found.length) return found.map(i => i.src || i.dataset.src).filter(Boolean);
}
return [];
}
""")
return result or []
except Exception:
return []
def _get_ext(url: str) -> str:
m = re.search(r"\.(jpg|jpeg|png|webp)(\?|$)", url, re.IGNORECASE)
if m:
ext = m.group(1).lower()
return ".jpg" if ext == "jpeg" else f".{ext}"
return ".jpg"
# ──────────────────────────────────────────────
# Скачивание главы
# ──────────────────────────────────────────────
async def get_chapter_images_and_download(
page: Page,
chapter_url: str,
dest_dir: Path,
manga_url: str | None = None,
on_page: object = None,
) -> list[Path]:
"""
1. Открывает страницу главы (устанавливает DDoS-Guard cookies для CDN).
2. Извлекает список URL из readerInit.
3. Перехватывает img-запросы через page.route() + route.fetch()
(браузерный стек — правильные Sec-Fetch-* заголовки, cookies).
4. Пролистывает читалку клавишей ArrowRight чтобы загрузить все страницы.
5. Retry для страниц с timeout через JS fetch.
"""
t_start = time.monotonic()
ch_id = chapter_url.split("/")[-1] # короткий идентификатор для логов
logger.info("[{}] Загружаем главу: {}", ch_id, chapter_url)
from urllib.parse import urlparse
parsed = urlparse(chapter_url)
parts = parsed.path.strip("/").split("/")
manga_slug = parts[0] if parts else ""
referer = manga_url or f"{parsed.scheme}://{parsed.netloc}/{manga_slug}"
load_url = chapter_url + ("?mtr=1" if "?" not in chapter_url else "&mtr=1")
dest_dir.mkdir(parents=True, exist_ok=True)
def _base(u: str) -> str:
return u.split("?")[0]
# Баннеры/рекламные изображения — игнорируем без логирования
BANNER_RE = re.compile(r"466_p\.|570_p\.|banner|advert", re.I)
def _is_manga_image(url: str) -> bool:
base = _base(url)
if not re.search(r"\.(jpg|jpeg|png|webp)(\?|$)", base, re.I):
return False
if "resrmr." in url or "/static/" in url:
return False
return bool(re.search(r"one-way\.work|staticfa\.|rm\.one-way|cdnmanga|reimg", url, re.I))
captured: dict[str, bytes] = {} # base_url → bytes
route_errors: dict[str, str] = {} # base_url → текст ошибки
route_statuses: dict[str, int] = {} # base_url → HTTP status (не 200/206)
lock = asyncio.Lock()
async def route_handler(route, request):
url = request.url
base = _base(url)
if not _is_manga_image(url):
await route.continue_()
return
if BANNER_RE.search(base):
await route.continue_()
return
async with lock:
already = base in captured
if already:
await route.continue_()
return
fname = base.split("/")[-1]
try:
response = await route.fetch()
status = response.status
body = await response.body()
if body and len(body) > 500 and status in (200, 206):
async with lock:
if base not in captured:
captured[base] = body
logger.debug("[{}] ✓ {}: {} байт", ch_id, fname, len(body))
if on_page:
try:
asyncio.ensure_future(on_page(0, 0))
except Exception:
pass
else:
async with lock:
route_statuses[base] = status
if status not in (200, 206):
logger.warning("[{}] CDN HTTP {} для '{}' | {}",
ch_id, status, fname, base[-70:])
else:
logger.warning("[{}] Слишком мал ответ ({} байт) для '{}'",
ch_id, len(body), fname)
await route.fulfill(response=response)
except Exception as e:
err = str(e)
async with lock:
route_errors[base] = err
is_timeout = "timeout" in err.lower()
level = logger.warning if is_timeout else logger.warning
level("[{}] route.fetch {} '{}': {}",
ch_id, "timeout" if is_timeout else "ошибка", fname, err[:150])
try:
await route.continue_()
except Exception:
pass
await page.route("**/*", route_handler)
# 1. Открываем главу
ok = await _navigate(page, load_url, referer=referer)
if not ok:
await page.unroute("**/*", route_handler)
logger.error("[{}] Не удалось открыть главу после всех retry: {}", ch_id, chapter_url)
return []
# 2. Ждём readerInit
try:
await page.wait_for_function(
"() => Array.from(document.querySelectorAll('script'))"
".some(s => s.textContent.includes('readerInit'))",
timeout=15_000,
)
except Exception as e:
logger.warning("[{}] readerInit не появился за 15с ({}). "
"Продолжаем через DOM-fallback.", ch_id, str(e)[:80])
# 3. Извлекаем список URL
image_urls = await _extract_images_from_js(page)
if not image_urls:
logger.debug("[{}] JS readerInit не дал URL, пробуем DOM-парсинг", ch_id)
image_urls = await _extract_images_from_dom(page)
if not image_urls:
await page.unroute("**/*", route_handler)
try:
page_info = await page.evaluate("() => document.title + ' | ' + location.href")
except Exception:
page_info = "?"
logger.error("[{}] Список изображений пуст. Текущая страница: {}", ch_id, page_info)
return []
logger.info("[{}] Найдено изображений: {}", ch_id, len(image_urls))
url_to_idx = {_base(u): i for i, u in enumerate(image_urls)}
filename_to_idx = {_base(u).split("/")[-1]: i for i, u in enumerate(image_urls)}
total = len(image_urls)
def _count_matched() -> int:
count = 0
for base_url in captured:
if base_url in url_to_idx or base_url.split("/")[-1] in filename_to_idx:
count += 1
return count
# 4. Пролистываем читалку
await asyncio.sleep(1)
stall_count = 0
prev_done = -1
for i in range(total + 20):
done = _count_matched()
if done >= total:
break
try:
await page.keyboard.press("ArrowRight")
await asyncio.sleep(0.5)
except Exception as e:
logger.warning("[{}] Ошибка листания на шаге {}: {}", ch_id, i + 1, e)
break
if i % 20 == 19:
done = _count_matched()
logger.debug("[{}] Пролистано {}, загружено: {}/{}", ch_id, i + 1, done, total)
if done == prev_done:
stall_count += 1
if stall_count >= 3:
logger.warning("[{}] Прогресс завис ({}/{}) после {} листаний — прерываем",
ch_id, done, total, i + 1)
break
else:
stall_count = 0
prev_done = done
# Финальное ожидание
await asyncio.sleep(3)
# 5. Retry для страниц с timeout через браузерный JS fetch
async with lock:
timeout_bases = [u for u, e in route_errors.items()
if "timeout" in e.lower() and u not in captured]
if timeout_bases:
logger.info("[{}] Retry {} страниц с timeout через JS fetch...",
ch_id, len(timeout_bases))
for retry_base in timeout_bases:
if retry_base in captured:
continue
fname = retry_base.split("/")[-1]
try:
data_b64 = await page.evaluate("""async (url) => {
try {
const r = await fetch(url, {credentials: 'include'});
if (!r.ok) return null;
const buf = await r.arrayBuffer();
const bytes = new Uint8Array(buf);
let bin = '';
for (let b of bytes) bin += String.fromCharCode(b);
return btoa(bin);
} catch(e) { return null; }
}""", retry_base)
if data_b64:
import base64
body = base64.b64decode(data_b64)
if len(body) > 500:
async with lock:
captured[retry_base] = body
logger.info("[{}] Retry OK: {} ({} байт)", ch_id, fname, len(body))
else:
logger.warning("[{}] Retry вернул {} байт для '{}' — игнорируем",
ch_id, len(body), fname)
else:
logger.warning("[{}] Retry вернул null для '{}' | {}",
ch_id, fname, retry_base[-70:])
except Exception as e2:
logger.warning("[{}] Retry JS ошибка для '{}': {}", ch_id, fname, e2)
await page.unroute("**/*", route_handler)
done = _count_matched()
elapsed = time.monotonic() - t_start
logger.info("[{}] Перехвачено: {}/{} за {:.1f}с", ch_id, done, total, elapsed)
# 6. Сохраняем в правильном порядке
filename_to_idx = {_base(u).split("/")[-1]: i for i, u in enumerate(image_urls)}
paths: dict[int, Path] = {}
unmatched_other: list[str] = []
for base_url, body in captured.items():
idx = url_to_idx.get(base_url)
if idx is None:
fname = base_url.split("/")[-1]
idx = filename_to_idx.get(fname)
if idx is None:
if not BANNER_RE.search(base_url):
unmatched_other.append(base_url.split("/")[-1])
continue
ext = _get_ext(base_url)
p = dest_dir / f"{idx:04d}{ext}"
p.write_bytes(body)
paths[idx] = p
if unmatched_other:
logger.debug("[{}] Перехвачено, но не совпало с readerInit ({}): {}",
ch_id, len(unmatched_other), unmatched_other)
# 7. Итоговый отчёт по пропущенным страницам
missing_idxs = [i for i in range(total) if i not in paths]
if missing_idxs:
missing_files = [_base(image_urls[i]).split("/")[-1] for i in missing_idxs]
missing_full = [_base(image_urls[i]) for i in missing_idxs]
timeout_miss = [missing_files[j] for j, i in enumerate(missing_idxs)
if missing_full[j] in route_errors
and "timeout" in route_errors[missing_full[j]].lower()]
http_miss = [f"{missing_files[j]}(HTTP {route_statuses.get(missing_full[j], '?')})"
for j, i in enumerate(missing_idxs)
if missing_full[j] in route_statuses]
unrcv = [missing_files[j] for j, i in enumerate(missing_idxs)
if missing_full[j] not in route_errors
and missing_full[j] not in route_statuses]
reasons = []
if timeout_miss:
reasons.append(f"timeout×{len(timeout_miss)}: {timeout_miss}")
if http_miss:
reasons.append(f"HTTP-err×{len(http_miss)}: {http_miss}")
if unrcv:
reasons.append(f"не_перехвачено×{len(unrcv)}: {unrcv}")
logger.warning(
"[{}] Пропущено {}/{} стр. | №: {} | причины: {}",
ch_id, len(missing_idxs), total,
[i + 1 for i in missing_idxs],
" | ".join(reasons) if reasons else "неизвестно",
)
logger.debug("[{}] Полные URL пропущенных: {}", ch_id, missing_full)
return [paths[i] for i in sorted(paths.keys())]

74
src/sources/__init__.py Normal file
View File

@@ -0,0 +1,74 @@
"""
Реестр источников манги.
Для добавления нового источника:
1. Создать файл src/sources/mysource.py с классом, реализующим MangaSourceProtocol
2. Импортировать его здесь и добавить в список SOURCES
"""
from urllib.parse import urlparse
from typing import Optional
from .base import MangaSourceProtocol
from .readmanga import ReadmangaSource
# ── Регистрация источников ─────────────────────
# Добавьте новые источники сюда:
SOURCES: list = [
ReadmangaSource(),
]
# Быстрый поиск по slug
_BY_SLUG: dict[str, object] = {s.slug: s for s in SOURCES}
class SourceRegistry:
"""Реестр источников. Источники определяются только в коде."""
def get_by_slug(self, slug: str) -> Optional[object]:
return _BY_SLUG.get(slug)
def get_by_db_id(self, source_id: int, db) -> Optional[object]:
"""Резолвит адаптер через БД: source_id → slug → экземпляр."""
row = db.get_source_by_id(source_id)
if not row:
return None
return _BY_SLUG.get(row["slug"])
def all_sources(self) -> list:
return list(SOURCES)
def all_slugs(self) -> list[str]:
return [s.slug for s in SOURCES]
registry = SourceRegistry()
def get_source_for_url(url: str, db) -> Optional[object]:
"""
Определяет источник по домену URL.
Ищет домен в таблице source_domains → возвращает адаптер.
Если домен не зарегистрирован — возвращает None.
"""
try:
domain = urlparse(url).netloc.lower()
if domain.startswith("www."):
domain = domain[4:]
row = db.get_source_by_domain(domain)
if not row:
return None
return _BY_SLUG.get(row["slug"])
except Exception:
return None
def extract_domain(url: str) -> str:
"""Извлекает домен без www."""
try:
domain = urlparse(url).netloc.lower()
if domain.startswith("www."):
domain = domain[4:]
return domain
except Exception:
return ""

58
src/sources/base.py Normal file
View File

@@ -0,0 +1,58 @@
"""
Базовые модели данных и Protocol-интерфейс для источников манги.
"""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Protocol, runtime_checkable
from playwright.async_api import Page
# ──────────────────────────────────────────────
# Модели данных (общие для всех источников)
# ──────────────────────────────────────────────
@dataclass
class Chapter:
title: str
url: str
number: float = 0.0
volume: int = 0
@dataclass
class MangaInfo:
title: str
url: str
chapters: list[Chapter] = field(default_factory=list)
pub_status: str = "unknown" # completed / ongoing / unknown
title_ru: str = ""
title_full: str = ""
description: str = ""
genres: list[str] = field(default_factory=list)
# ──────────────────────────────────────────────
# Интерфейс источника
# ──────────────────────────────────────────────
@runtime_checkable
class MangaSourceProtocol(Protocol):
slug: str # уникальный код источника в коде ("readmanga")
display_name: str # название для UI ("ReadManga")
async def get_manga_info(self, page: Page, url: str) -> Optional[MangaInfo]:
"""Возвращает информацию о манге и список глав."""
...
async def get_chapter_images_and_download(
self,
page: Page,
chapter_url: str,
dest_dir: Path,
manga_url: Optional[str] = None,
on_page: object = None,
) -> list[Path]:
"""Скачивает страницы главы в dest_dir и возвращает список путей."""
...

589
src/sources/readmanga.py Normal file
View File

@@ -0,0 +1,589 @@
"""
Адаптер ReadManga: поддерживает readmanga.ru и все его клоны.
"""
import asyncio
import base64
import re
import time
from pathlib import Path
from typing import Optional
from loguru import logger
from playwright.async_api import Page
from .base import Chapter, MangaInfo
class ReadmangaSource:
slug = "readmanga"
display_name = "ReadManga"
# CDN-домены из которых принимаем картинки глав
cdn_patterns = ["one-way.work", "staticfa.", "rm.one-way", "cdnmanga", "reimg"]
# ──────────────────────────────────────────────
# Страница манги — список глав
# ──────────────────────────────────────────────
async def get_manga_info(self, page: Page, url: str) -> Optional[MangaInfo]:
"""Открывает страницу манги и возвращает список всех глав."""
logger.info("Загружаем страницу манги: {}", url)
ok = await _navigate(page, url)
if not ok:
return None
title_full = await page.title()
title_full = re.sub(r"\s*[-–|].*$", "", title_full).strip()
title_ru = await _extract_ru_title_from_dom(page)
if not title_ru:
title_ru = _parse_ru_title(title_full)
logger.info("Манга: {} | ru: {}", title_full, title_ru)
pub_status = await _extract_pub_status(page)
logger.info("Статус выпуска: {}", pub_status)
description = await _extract_description(page)
genres = await _extract_genres(page)
await _expand_chapters(page)
chapters = await _extract_chapters(page)
if not chapters:
chapters = await _extract_chapters_alt(page)
logger.info("Найдено глав: {}", len(chapters))
return MangaInfo(
title=title_ru or title_full,
url=url,
chapters=chapters,
pub_status=pub_status,
title_ru=title_ru,
title_full=title_full,
description=description,
genres=genres,
)
# ──────────────────────────────────────────────
# Скачивание главы
# ──────────────────────────────────────────────
async def get_chapter_images_and_download(
self,
page: Page,
chapter_url: str,
dest_dir: Path,
manga_url: Optional[str] = None,
on_page: object = None,
) -> list[Path]:
"""
1. Открывает страницу главы.
2. Извлекает список URL из readerInit.
3. Перехватывает img-запросы через page.route().
4. Пролистывает читалку клавишей ArrowRight.
5. Retry для страниц с timeout через JS fetch.
"""
cdn_patterns = self.cdn_patterns
t_start = time.monotonic()
ch_id = chapter_url.split("/")[-1]
logger.info("[{}] Загружаем главу: {}", ch_id, chapter_url)
from urllib.parse import urlparse
parsed = urlparse(chapter_url)
parts = parsed.path.strip("/").split("/")
manga_slug = parts[0] if parts else ""
referer = manga_url or f"{parsed.scheme}://{parsed.netloc}/{manga_slug}"
load_url = chapter_url + ("?mtr=1" if "?" not in chapter_url else "&mtr=1")
dest_dir.mkdir(parents=True, exist_ok=True)
def _base(u: str) -> str:
return u.split("?")[0]
BANNER_RE = re.compile(r"466_p\.|570_p\.|banner|advert", re.I)
def _is_manga_image(url: str) -> bool:
base = _base(url)
if not re.search(r"\.(jpg|jpeg|png|webp)(\?|$)", base, re.I):
return False
if "resrmr." in url or "/static/" in url:
return False
pattern = "|".join(re.escape(p) for p in cdn_patterns)
return bool(re.search(pattern, url, re.I))
captured: dict[str, bytes] = {}
route_errors: dict[str, str] = {}
route_statuses: dict[str, int] = {}
lock = asyncio.Lock()
async def route_handler(route, request):
url = request.url
base = _base(url)
if not _is_manga_image(url):
await route.continue_()
return
if BANNER_RE.search(base):
await route.continue_()
return
async with lock:
already = base in captured
if already:
await route.continue_()
return
fname = base.split("/")[-1]
try:
response = await route.fetch()
status = response.status
body = await response.body()
if body and len(body) > 500 and status in (200, 206):
async with lock:
if base not in captured:
captured[base] = body
logger.debug("[{}] ✓ {}: {} байт", ch_id, fname, len(body))
if on_page:
try:
asyncio.ensure_future(on_page(0, 0))
except Exception:
pass
else:
async with lock:
route_statuses[base] = status
if status not in (200, 206):
logger.warning("[{}] CDN HTTP {} для '{}' | {}",
ch_id, status, fname, base[-70:])
else:
logger.warning("[{}] Слишком мал ответ ({} байт) для '{}'",
ch_id, len(body), fname)
await route.fulfill(response=response)
except Exception as e:
err = str(e)
async with lock:
route_errors[base] = err
is_timeout = "timeout" in err.lower()
logger.warning("[{}] route.fetch {} '{}': {}",
ch_id, "timeout" if is_timeout else "ошибка", fname, err[:150])
try:
await route.continue_()
except Exception:
pass
await page.route("**/*", route_handler)
ok = await _navigate(page, load_url, referer=referer)
if not ok:
await page.unroute("**/*", route_handler)
logger.error("[{}] Не удалось открыть главу: {}", ch_id, chapter_url)
return []
try:
await page.wait_for_function(
"() => Array.from(document.querySelectorAll('script'))"
".some(s => s.textContent.includes('readerInit'))",
timeout=15_000,
)
except Exception as e:
logger.warning("[{}] readerInit не появился за 15с ({}). DOM-fallback.", ch_id, str(e)[:80])
image_urls = await _extract_images_from_js(page)
if not image_urls:
logger.debug("[{}] JS readerInit не дал URL, пробуем DOM-парсинг", ch_id)
image_urls = await _extract_images_from_dom(page)
if not image_urls:
await page.unroute("**/*", route_handler)
try:
page_info = await page.evaluate("() => document.title + ' | ' + location.href")
except Exception:
page_info = "?"
logger.error("[{}] Список изображений пуст. Страница: {}", ch_id, page_info)
return []
logger.info("[{}] Найдено изображений: {}", ch_id, len(image_urls))
url_to_idx = {_base(u): i for i, u in enumerate(image_urls)}
filename_to_idx = {_base(u).split("/")[-1]: i for i, u in enumerate(image_urls)}
total = len(image_urls)
def _count_matched() -> int:
count = 0
for base_url in captured:
if base_url in url_to_idx or base_url.split("/")[-1] in filename_to_idx:
count += 1
return count
await asyncio.sleep(1)
stall_count = 0
prev_done = -1
for i in range(total + 20):
done = _count_matched()
if done >= total:
break
try:
await page.keyboard.press("ArrowRight")
await asyncio.sleep(0.5)
except Exception as e:
logger.warning("[{}] Ошибка листания на шаге {}: {}", ch_id, i + 1, e)
break
if i % 20 == 19:
done = _count_matched()
logger.debug("[{}] Пролистано {}, загружено: {}/{}", ch_id, i + 1, done, total)
if done == prev_done:
stall_count += 1
if stall_count >= 3:
logger.warning("[{}] Прогресс завис ({}/{}) — прерываем", ch_id, done, total)
break
else:
stall_count = 0
prev_done = done
await asyncio.sleep(3)
# Retry timeout через JS fetch
async with lock:
timeout_bases = [u for u, e in route_errors.items()
if "timeout" in e.lower() and u not in captured]
if timeout_bases:
logger.info("[{}] Retry {} страниц с timeout...", ch_id, len(timeout_bases))
for retry_base in timeout_bases:
if retry_base in captured:
continue
fname = retry_base.split("/")[-1]
try:
data_b64 = await page.evaluate("""async (url) => {
try {
const r = await fetch(url, {credentials: 'include'});
if (!r.ok) return null;
const buf = await r.arrayBuffer();
const bytes = new Uint8Array(buf);
let bin = '';
for (let b of bytes) bin += String.fromCharCode(b);
return btoa(bin);
} catch(e) { return null; }
}""", retry_base)
if data_b64:
body = base64.b64decode(data_b64)
if len(body) > 500:
async with lock:
captured[retry_base] = body
logger.info("[{}] Retry OK: {} ({} байт)", ch_id, fname, len(body))
else:
logger.warning("[{}] Retry вернул {} байт — игнорируем", ch_id, len(body))
else:
logger.warning("[{}] Retry null для '{}'", ch_id, fname)
except Exception as e2:
logger.warning("[{}] Retry JS ошибка '{}': {}", ch_id, fname, e2)
await page.unroute("**/*", route_handler)
done = _count_matched()
elapsed = time.monotonic() - t_start
logger.info("[{}] Перехвачено: {}/{} за {:.1f}с", ch_id, done, total, elapsed)
filename_to_idx = {_base(u).split("/")[-1]: i for i, u in enumerate(image_urls)}
paths: dict[int, Path] = {}
unmatched_other: list[str] = []
for base_url, body in captured.items():
idx = url_to_idx.get(base_url)
if idx is None:
fname = base_url.split("/")[-1]
idx = filename_to_idx.get(fname)
if idx is None:
if not BANNER_RE.search(base_url):
unmatched_other.append(base_url.split("/")[-1])
continue
ext = _get_ext(base_url)
p = dest_dir / f"{idx:04d}{ext}"
p.write_bytes(body)
paths[idx] = p
if unmatched_other:
logger.debug("[{}] Не совпало с readerInit ({}): {}", ch_id, len(unmatched_other), unmatched_other)
missing_idxs = [i for i in range(total) if i not in paths]
if missing_idxs:
missing_files = [_base(image_urls[i]).split("/")[-1] for i in missing_idxs]
missing_full = [_base(image_urls[i]) for i in missing_idxs]
timeout_miss = [missing_files[j] for j, i in enumerate(missing_idxs)
if missing_full[j] in route_errors
and "timeout" in route_errors[missing_full[j]].lower()]
http_miss = [f"{missing_files[j]}(HTTP {route_statuses.get(missing_full[j], '?')})"
for j, i in enumerate(missing_idxs)
if missing_full[j] in route_statuses]
unrcv = [missing_files[j] for j, i in enumerate(missing_idxs)
if missing_full[j] not in route_errors
and missing_full[j] not in route_statuses]
reasons = []
if timeout_miss:
reasons.append(f"timeout×{len(timeout_miss)}: {timeout_miss}")
if http_miss:
reasons.append(f"HTTP-err×{len(http_miss)}: {http_miss}")
if unrcv:
reasons.append(f"не_перехвачено×{len(unrcv)}: {unrcv}")
logger.warning(
"[{}] Пропущено {}/{} стр. | №: {} | причины: {}",
ch_id, len(missing_idxs), total,
[i + 1 for i in missing_idxs],
" | ".join(reasons) if reasons else "неизвестно",
)
return [paths[i] for i in sorted(paths.keys())]
# ──────────────────────────────────────────────
# Вспомогательные функции (приватные)
# ──────────────────────────────────────────────
async def _navigate(page: Page, url: str, retries: int = 3,
referer: str | None = None) -> bool:
from urllib.parse import urlparse
if referer is None:
p = urlparse(url)
referer = f"{p.scheme}://{p.netloc}/"
for attempt in range(1, retries + 1):
try:
resp = await page.goto(url, wait_until="domcontentloaded",
timeout=60_000, referer=referer)
if resp and resp.status >= 400:
logger.warning("Попытка {}/{}: HTTP {}", attempt, retries, resp.status)
await asyncio.sleep(3 * attempt)
continue
try:
await page.wait_for_load_state("networkidle", timeout=10_000)
except Exception:
pass
return True
except Exception as e:
logger.warning("Попытка {}/{}: {}", attempt, retries, e)
await asyncio.sleep(3 * attempt)
return False
async def _extract_ru_title_from_dom(page: Page) -> str:
try:
result = await page.evaluate("""
() => {
const selectors = [
'.names .name', 'h1.manga-title', 'h1 .name', '.name-block .name',
];
for (const sel of selectors) {
const el = document.querySelector(sel);
if (el && el.textContent.trim()) return el.textContent.trim();
}
return '';
}
""")
return (result or "").strip()
except Exception:
return ""
def _parse_ru_title(full_title: str) -> str:
t = full_title.strip()
t = re.sub(r'^Манга\s+', '', t).strip()
t = re.split(r'\s*[\(\[]', t)[0].strip()
t = re.sub(r'\s+онлайн\s*$', '', t, flags=re.IGNORECASE).strip()
words = t.split()
result = []
for w in words:
if re.search(r'[а-яёА-ЯЁ]', w):
result.append(w)
elif re.search(r'[a-zA-Z]', w):
if result:
break
else:
if result:
result.append(w)
while result and not re.search(r'[а-яёА-ЯЁ]', result[-1]):
result.pop()
if result:
t = ' '.join(result)
return t
async def _extract_pub_status(page: Page) -> str:
try:
result = await page.evaluate("""
() => {
const statusSelectors = [
'.elem_status .value', '.manga-info .status',
'[class*="status"] .value', '.property .status',
];
for (const sel of statusSelectors) {
const el = document.querySelector(sel);
if (el) {
const t = el.textContent.toLowerCase();
if (t.includes('завершён') || t.includes('завершен') || t.includes('complete')) return 'completed';
if (t.includes('продолжает') || t.includes('ongoing')) return 'ongoing';
}
}
const bodyText = document.body ? document.body.innerText.toLowerCase() : '';
if (bodyText.includes('выпуск завершён') || bodyText.includes('выпуск завершен')) return 'completed';
if (bodyText.includes('продолжается')) return 'ongoing';
return 'unknown';
}
""")
return result or "unknown"
except Exception:
return "unknown"
async def _extract_description(page: Page) -> str:
try:
result = await page.evaluate("""
() => {
const selectors = [
'.manga-description', '.elem_descr .value',
'#tab-description .description-text', '.description',
'[itemprop="description"]',
];
for (const sel of selectors) {
const el = document.querySelector(sel);
if (el && el.textContent.trim()) return el.textContent.trim();
}
return '';
}
""")
return (result or "").strip()[:2000]
except Exception:
return ""
async def _extract_genres(page: Page) -> list[str]:
try:
result = await page.evaluate("""
() => {
const selectors = [
'.elem_genre .value a', '.genres a',
'[itemprop="genre"]', '.genre-list a',
];
for (const sel of selectors) {
const els = document.querySelectorAll(sel);
if (els.length) return Array.from(els).map(e => e.textContent.trim()).filter(Boolean);
}
return [];
}
""")
return result or []
except Exception:
return []
async def _expand_chapters(page: Page):
for sel in ["a.chapter-link.all", "button:has-text('Все главы')", "a:has-text('Все главы')"]:
try:
el = page.locator(sel).first
if await el.is_visible(timeout=2000):
await el.click()
await page.wait_for_load_state("networkidle", timeout=10_000)
return
except Exception:
pass
async def _extract_chapters(page: Page) -> list[Chapter]:
rows = await page.query_selector_all("#chapters-list tr.item-row")
chapters = []
for row in rows:
link = await row.query_selector("td[class*='item-title'] a")
if not link:
continue
href = await link.get_attribute("href") or ""
text = (await link.inner_text()).strip()
if not href:
continue
td = await row.query_selector("td[data-num]")
vol = int((await td.get_attribute("data-vol") or "0")) if td else 0
num_raw = int((await td.get_attribute("data-num") or "0")) if td else 0
number = num_raw / 10.0
full_url = href if href.startswith("http") else _base_url(page.url) + href
chapters.append(Chapter(title=text, url=full_url, number=number, volume=vol))
return chapters
async def _extract_chapters_alt(page: Page) -> list[Chapter]:
result = await page.evaluate("""
() => {
const links = Array.from(document.querySelectorAll('a[href*="/vol"]'));
return links.map(a => ({ href: a.href, text: a.textContent.trim() }))
.filter(x => x.href && x.text);
}
""")
return [Chapter(title=x["text"], url=x["href"],
number=_parse_num(x["text"]), volume=_parse_vol(x["text"]))
for x in result]
async def _extract_images_from_js(page: Page) -> list[str]:
try:
result = await page.evaluate("""
() => {
for (const s of document.querySelectorAll('script')) {
const text = s.textContent || '';
const mi = text.indexOf('readerInit');
if (mi === -1) continue;
const ai = text.indexOf('[', mi);
if (ai === -1) continue;
let depth = 0, end = -1;
for (let i = ai; i < text.length; i++) {
if (text[i] === '[') depth++;
else if (text[i] === ']') { depth--; if (!depth) { end = i+1; break; } }
}
if (end === -1) continue;
try {
const arr = eval(text.slice(ai, end));
if (Array.isArray(arr) && arr.length)
return arr.map(item => Array.isArray(item) && item.length >= 3
? item[0] + item[2] : null).filter(Boolean);
} catch(e) {}
}
return [];
}
""")
if result:
logger.debug("JS readerInit нашёл {} изображений", len(result))
return result or []
except Exception as e:
logger.debug("JS-метод не сработал: {}", e)
return []
async def _extract_images_from_dom(page: Page) -> list[str]:
try:
result = await page.evaluate("""
() => {
for (const sel of ['img.manga-page', '.page-image img', '#mangaReader img', 'img[data-src]']) {
const found = Array.from(document.querySelectorAll(sel));
if (found.length) return found.map(i => i.src || i.dataset.src).filter(Boolean);
}
return [];
}
""")
return result or []
except Exception:
return []
def _get_ext(url: str) -> str:
m = re.search(r"\.(jpg|jpeg|png|webp)(\?|$)", url, re.IGNORECASE)
if m:
ext = m.group(1).lower()
return ".jpg" if ext == "jpeg" else f".{ext}"
return ".jpg"
def _base_url(url: str) -> str:
m = re.match(r"(https?://[^/]+)", url)
return m.group(1) if m else "https://readmanga.ru"
def _parse_num(text: str) -> float:
m = re.search(r"[\d]+(?:[.,]\d+)?", text.replace(",", "."))
return float(m.group()) if m else 0.0
def _parse_vol(text: str) -> int:
m = re.search(r"Том\s+(\d+)", text, re.IGNORECASE)
return int(m.group(1)) if m else 0

235
src/state.py
View File

@@ -1,14 +1,25 @@
"""
Хранение состояния скачивания в SQLite.
"""
import json
import sqlite3
from datetime import datetime
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse
DB_PATH = Path("/app/state/progress.db")
# Домены ReadManga по умолчанию (сидинг при первом запуске)
_DEFAULT_READMANGA_DOMAINS = [
"readmanga.ru",
"readmanga.live",
"readmanga.me",
"readmanga.io",
"3.readmanga.ru",
]
class StateDB:
def __init__(self, db_path: Path = DB_PATH):
@@ -68,18 +79,35 @@ class StateDB:
created_at TEXT
)
""")
self.conn.execute("""
CREATE TABLE IF NOT EXISTS sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
slug TEXT UNIQUE NOT NULL,
display_name TEXT NOT NULL,
settings TEXT DEFAULT '{}',
created_at TEXT
)
""")
self.conn.execute("""
CREATE TABLE IF NOT EXISTS source_domains (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_id INTEGER NOT NULL REFERENCES sources(id),
domain TEXT UNIQUE NOT NULL
)
""")
# Migrate old DB: add missing columns
migrations = [
("chapters", "pages_total", "INTEGER DEFAULT 0"),
("chapters", "pages_done", "INTEGER DEFAULT 0"),
("mangas", "title_ru", "TEXT"),
("mangas", "title_full", "TEXT"),
("mangas", "pub_status", "TEXT DEFAULT 'unknown'"),
("mangas", "auto_update", "INTEGER DEFAULT 0"),
("mangas", "last_checked_at", "TEXT"),
("mangas", "started_at", "TEXT"),
("mangas", "finished_at", "TEXT"),
("mangas", "folder_name", "TEXT"),
("chapters", "pages_total", "INTEGER DEFAULT 0"),
("chapters", "pages_done", "INTEGER DEFAULT 0"),
("mangas", "title_ru", "TEXT"),
("mangas", "title_full", "TEXT"),
("mangas", "pub_status", "TEXT DEFAULT 'unknown'"),
("mangas", "auto_update", "INTEGER DEFAULT 0"),
("mangas", "last_checked_at","TEXT"),
("mangas", "started_at", "TEXT"),
("mangas", "finished_at", "TEXT"),
("mangas", "folder_name", "TEXT"),
("mangas", "source_id", "INTEGER REFERENCES sources(id)"),
]
for table, col, typedef in migrations:
try:
@@ -88,17 +116,184 @@ class StateDB:
pass
self.conn.commit()
def sync_sources(self, registry) -> None:
"""
Синхронизирует таблицу sources с реестром из кода.
Вызывается при старте приложения.
При первом запуске создаёт записи и засеивает домены ReadManga.
"""
from loguru import logger
for source in registry.all_sources():
existing = self.conn.execute(
"SELECT id, display_name FROM sources WHERE slug=?", (source.slug,)
).fetchone()
if not existing:
self.conn.execute(
"INSERT INTO sources (slug, display_name, settings, created_at) VALUES (?,?,?,?)",
(source.slug, source.display_name, "{}", _now())
)
logger.info("Источник добавлен в БД: {} ({})", source.display_name, source.slug)
else:
if existing["display_name"] != source.display_name:
self.conn.execute(
"UPDATE sources SET display_name=? WHERE slug=?",
(source.display_name, source.slug)
)
self.conn.commit()
# Сидинг доменов ReadManga при первом запуске
rm = self.conn.execute("SELECT id FROM sources WHERE slug='readmanga'").fetchone()
if rm:
count = self.conn.execute(
"SELECT COUNT(*) FROM source_domains WHERE source_id=?", (rm["id"],)
).fetchone()[0]
if count == 0:
for domain in _DEFAULT_READMANGA_DOMAINS:
try:
self.conn.execute(
"INSERT INTO source_domains (source_id, domain) VALUES (?,?)",
(rm["id"], domain)
)
except Exception:
pass
self.conn.commit()
logger.info("Сидинг доменов ReadManga: {} доменов", len(_DEFAULT_READMANGA_DOMAINS))
# Логируем источники в БД без кода (не в реестре)
known_slugs = set(registry.all_slugs())
db_slugs = [r["slug"] for r in self.conn.execute("SELECT slug FROM sources").fetchall()]
for slug in db_slugs:
if slug not in known_slugs:
logger.warning("Источник '{}' есть в БД, но отсутствует в реестре — манги недоступны", slug)
def migrate_manga_sources(self) -> int:
"""
Авто-миграция: проставляет source_id для манг с source_id IS NULL.
Определяет источник по домену URL через source_domains.
Возвращает количество обновлённых манг.
"""
nulls = self.conn.execute(
"SELECT url FROM mangas WHERE source_id IS NULL"
).fetchall()
updated = 0
for row in nulls:
url = row["url"]
domain = _extract_domain(url)
source_row = self.get_source_by_domain(domain)
if source_row:
self.conn.execute(
"UPDATE mangas SET source_id=? WHERE url=?",
(source_row["id"], url)
)
updated += 1
if updated:
self.conn.commit()
return updated
# ── Sources ───────────────────────────────────
def get_source_by_id(self, source_id: int) -> Optional[dict]:
row = self.conn.execute("SELECT * FROM sources WHERE id=?", (source_id,)).fetchone()
return dict(row) if row else None
def get_source_by_slug(self, slug: str) -> Optional[dict]:
row = self.conn.execute("SELECT * FROM sources WHERE slug=?", (slug,)).fetchone()
return dict(row) if row else None
def get_source_by_domain(self, domain: str) -> Optional[dict]:
"""Возвращает запись source по домену (через source_domains JOIN)."""
row = self.conn.execute("""
SELECT s.* FROM sources s
JOIN source_domains sd ON sd.source_id = s.id
WHERE sd.domain=?
""", (domain.lower(),)).fetchone()
return dict(row) if row else None
def get_all_sources(self) -> list[dict]:
"""Возвращает все источники с вложенным списком доменов."""
sources = self.conn.execute("SELECT * FROM sources ORDER BY id").fetchall()
result = []
for s in sources:
s_dict = dict(s)
domains = self.conn.execute(
"SELECT domain FROM source_domains WHERE source_id=? ORDER BY domain",
(s["id"],)
).fetchall()
s_dict["domains"] = [d["domain"] for d in domains]
try:
s_dict["settings"] = json.loads(s_dict.get("settings") or "{}")
except Exception:
s_dict["settings"] = {}
result.append(s_dict)
return result
def add_domain(self, source_id: int, domain: str) -> bool:
"""Добавляет домен к источнику. Возвращает False если уже существует."""
domain = domain.lower().strip()
try:
self.conn.execute(
"INSERT INTO source_domains (source_id, domain) VALUES (?,?)",
(source_id, domain)
)
self.conn.commit()
return True
except Exception:
return False
def remove_domain(self, source_id: int, domain: str) -> bool:
"""Удаляет домен у источника. Возвращает True если удалён."""
cur = self.conn.execute(
"DELETE FROM source_domains WHERE source_id=? AND domain=?",
(source_id, domain.lower())
)
self.conn.commit()
return cur.rowcount > 0
def set_manga_source(self, manga_url: str, source_id: int) -> None:
"""Меняет источник у манги."""
self.conn.execute(
"UPDATE mangas SET source_id=?, updated_at=? WHERE url=?",
(source_id, _now(), manga_url)
)
self.conn.commit()
def reset_failed_chapters(self, manga_url: str) -> int:
"""Сбрасывает failed и partial главы в pending. Возвращает количество."""
now = _now()
c1 = self.conn.execute(
"UPDATE chapters SET status='pending', pages_done=0, pages_total=0, updated_at=? "
"WHERE manga_url=? AND status='failed'",
(now, manga_url)
).rowcount
c2 = self.conn.execute(
"""UPDATE chapters SET status='pending', pages_done=0, pages_total=0, updated_at=?
WHERE manga_url=? AND status='done'
AND pages_total > 0 AND pages_done < pages_total""",
(now, manga_url)
).rowcount
self.conn.commit()
return c1 + c2
def count_mangas_by_source_domain(self, domain: str) -> int:
"""Считает манги с указанным доменом (для предупреждений в UI)."""
source = self.get_source_by_domain(domain)
if not source:
return 0
return self.conn.execute(
"SELECT COUNT(*) FROM mangas WHERE source_id=?", (source["id"],)
).fetchone()[0]
# ── Mangas ────────────────────────────────────
def add_manga(self, url: str, fmt: str = "cbz") -> bool:
def add_manga(self, url: str, fmt: str = "cbz", source_id: Optional[int] = None) -> bool:
"""Добавляет мангу в очередь. Возвращает True если новая."""
cur = self.conn.execute("SELECT id FROM mangas WHERE url=?", (url,))
if cur.fetchone():
return False
self.conn.execute("""
INSERT INTO mangas (url, format, status, added_at, updated_at)
VALUES (?, ?, 'queued', ?, ?)
""", (url, fmt, _now(), _now()))
INSERT INTO mangas (url, format, status, source_id, added_at, updated_at)
VALUES (?, ?, 'queued', ?, ?, ?)
""", (url, fmt, source_id, _now(), _now()))
self.conn.commit()
return True
@@ -318,3 +513,15 @@ class StateDB:
def _now() -> str:
return datetime.utcnow().isoformat()
def _extract_domain(url: str) -> str:
"""Извлекает домен без www."""
try:
domain = urlparse(url).netloc.lower()
if domain.startswith("www."):
domain = domain[4:]
return domain
except Exception:
return ""

33
src/worker.py
View File

@@ -11,7 +11,9 @@ from typing import Callable, Optional
from loguru import logger
from .browser import BrowserManager
from .scraper import get_manga_info, get_chapter_images_and_download, Chapter
from .sources import registry, get_source_for_url, extract_domain
from .sources.base import Chapter, MangaInfo
from .scraper import get_manga_info, get_chapter_images_and_download # shim для обратной совместимости
from .exporter import export, MangaMeta
from .state import StateDB
@@ -61,10 +63,23 @@ async def download_manga(
started_ts = await db_call(db.mark_started, url)
await emit({"type": "manga_start", "url": url, "started_at": started_ts})
# Резолвим источник
source = get_source_for_url(url, db)
if source is None:
# Последний шанс: по source_id в БД
manga_row = await db_call(db.get_manga, url)
if manga_row and manga_row.get("source_id"):
source = registry.get_by_db_id(manga_row["source_id"], db)
if source is None:
await db_call(db.update_manga_status, url, "failed")
await emit({"type": "source_unknown", "url": url,
"error": "Источник не определён. Выберите источник в настройках манги."})
return
async with BrowserManager(headless=True) as bm:
ctx, info_page = await bm.new_page()
manga = await get_manga_info(info_page, url)
manga = await source.get_manga_info(info_page, url)
await info_page.close()
if not manga:
@@ -193,7 +208,7 @@ async def download_manga(
"pages_total": pages_total,
})
image_paths = await get_chapter_images_and_download(
image_paths = await source.get_chapter_images_and_download(
ch_page, ch.url,
dest_dir=tmp_path,
manga_url=url,
@@ -329,9 +344,19 @@ async def check_for_updates(
db.add_history(manga_url=url, event_type="check_started")
await emit({"type": "check_started", "url": url})
# Резолвим источник
source = get_source_for_url(url, db)
if source is None:
manga_row = db.get_manga(url)
if manga_row and manga_row.get("source_id"):
source = registry.get_by_db_id(manga_row["source_id"], db)
if source is None:
await emit({"type": "source_unknown", "url": url})
return []
async with BrowserManager(headless=True) as bm:
_, page = await bm.new_page()
manga = await get_manga_info(page, url)
manga = await source.get_manga_info(page, url)
await page.close()
if not manga:
return []