upd
This commit is contained in:
80
src/api.py
80
src/api.py
@@ -4,7 +4,7 @@ FastAPI веб-сервер: REST API + WebSocket для мониторинга
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
@@ -16,9 +16,11 @@ from pydantic import BaseModel
|
||||
from loguru import logger
|
||||
from .state import StateDB
|
||||
from .worker import download_manga, check_for_updates
|
||||
from .browser import BrowserManager
|
||||
from .exporter import patch_meta, MangaMeta
|
||||
from .sources import registry, get_source_for_url, extract_domain
|
||||
from .auth import verify_password, hash_password, generate_session_token, COOKIE_NAME, COOKIE_MAX_AGE
|
||||
from .utils import safe_name
|
||||
OUTPUT_DIR = Path("/app/output")
|
||||
FRONTEND_DIR = Path("/app/frontend")
|
||||
app = FastAPI(title="Manga Downloader API")
|
||||
@@ -247,13 +249,11 @@ async def _run_auto_updates():
|
||||
except Exception as e:
|
||||
logger.error("Ошибка авто-обновления {}: {}", url, e)
|
||||
# ── Helpers ───────────────────────────────────
|
||||
def _safe_name(s: str) -> str:
|
||||
return re.sub(r'[^\w\s\-]', '', s).strip().replace(" ", "_")[:80]
|
||||
def _manga_folder(m: dict) -> Path:
|
||||
if m.get("folder_name"):
|
||||
return OUTPUT_DIR / m["folder_name"]
|
||||
title = m.get("title") or ""
|
||||
return OUTPUT_DIR / _safe_name(title)
|
||||
return OUTPUT_DIR / safe_name(title)
|
||||
def _dir_size(path: Path) -> int:
|
||||
if not path.exists():
|
||||
return 0
|
||||
@@ -266,16 +266,7 @@ def _format_size(bytes_val: int) -> str:
|
||||
return f"{bytes_val:.1f} ТБ"
|
||||
def _enrich_manga(m: dict, db: StateDB) -> dict:
|
||||
size_bytes = _dir_size(_manga_folder(m))
|
||||
ch_done_count = db.conn.execute(
|
||||
"SELECT COUNT(*) FROM chapters WHERE manga_url=? AND status='done'", (m["url"],)
|
||||
).fetchone()[0]
|
||||
ch_failed = db.conn.execute(
|
||||
"SELECT COUNT(*) FROM chapters WHERE manga_url=? AND status='failed'", (m["url"],)
|
||||
).fetchone()[0]
|
||||
ch_partial = db.conn.execute(
|
||||
"SELECT COUNT(*) FROM chapters WHERE manga_url=? AND status='done'"
|
||||
" AND pages_total > 0 AND pages_done < pages_total", (m["url"],)
|
||||
).fetchone()[0]
|
||||
stats = db.get_chapter_stats(m["url"])
|
||||
source_info = None
|
||||
if m.get("source_id"):
|
||||
src = db.get_source_by_id(m["source_id"])
|
||||
@@ -285,12 +276,12 @@ def _enrich_manga(m: dict, db: StateDB) -> dict:
|
||||
source_info = {"id": m["source_id"], "slug": "unknown", "display_name": "Источник недоступен"}
|
||||
return {
|
||||
**m,
|
||||
"chapters_done": ch_done_count,
|
||||
"chapters_done": stats["done"],
|
||||
"size_bytes": size_bytes,
|
||||
"size_human": _format_size(size_bytes),
|
||||
"queue_position": None,
|
||||
"is_active": m["url"] in active_tasks,
|
||||
"errors_count": ch_failed + ch_partial,
|
||||
"errors_count": stats["failed"] + stats["partial"],
|
||||
"started_at": m.get("started_at"),
|
||||
"finished_at": m.get("finished_at"),
|
||||
"source": source_info,
|
||||
@@ -366,7 +357,7 @@ async def login(body: LoginRequest, response: Response):
|
||||
if not user or not verify_password(body.password, user["password"]):
|
||||
raise HTTPException(status_code=401, detail="Неверный логин или пароль")
|
||||
token = generate_session_token()
|
||||
expires_at = (datetime.utcnow() + timedelta(days=30)).isoformat()
|
||||
expires_at = (datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30)).isoformat()
|
||||
db.create_session(token, user["id"], expires_at)
|
||||
response.set_cookie(key=COOKIE_NAME, value=token, max_age=COOKIE_MAX_AGE,
|
||||
httponly=True, samesite="lax", secure=False)
|
||||
@@ -483,11 +474,9 @@ async def list_mangas(_: dict = Depends(get_current_user)):
|
||||
try:
|
||||
mangas = db.get_all_mangas()
|
||||
result = [_enrich_manga(m, db) for m in mangas]
|
||||
queue_list = list(download_queue._queue)
|
||||
for i, job in enumerate(queue_list):
|
||||
for r in result:
|
||||
if r["url"] == job["url"]:
|
||||
r["queue_position"] = i + 1
|
||||
queue_positions = {job["url"]: i + 1 for i, job in enumerate(download_queue._queue)}
|
||||
for r in result:
|
||||
r["queue_position"] = queue_positions.get(r["url"])
|
||||
return result
|
||||
finally:
|
||||
db.close()
|
||||
@@ -545,7 +534,6 @@ async def add_to_queue(body: AddMangaRequest, current_user: dict = Depends(get_c
|
||||
return {"added": added, "skipped": skipped}
|
||||
async def _fetch_preview(url: str):
|
||||
try:
|
||||
from .browser import BrowserManager
|
||||
db = StateDB()
|
||||
try:
|
||||
source = get_source_for_url(url, db)
|
||||
@@ -621,13 +609,7 @@ async def _check_and_queue(url: str):
|
||||
async def get_news(limit: int = 100, _: dict = Depends(get_current_user)):
|
||||
db = StateDB()
|
||||
try:
|
||||
cur = db.conn.execute("""
|
||||
SELECT h.*, m.title as manga_title, m.title_ru
|
||||
FROM history h LEFT JOIN mangas m ON h.manga_url = m.url
|
||||
WHERE h.event_type IN ('downloaded', 'auto_downloaded')
|
||||
ORDER BY h.created_at DESC LIMIT ?
|
||||
""", (limit,))
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
return db.get_news(limit)
|
||||
finally:
|
||||
db.close()
|
||||
@app.get("/api/history")
|
||||
@@ -677,15 +659,7 @@ async def retry_errors(url: str, current_user: dict = Depends(get_current_user))
|
||||
if not manga:
|
||||
raise HTTPException(status_code=404, detail="Манга не найдена")
|
||||
_check_manga_access(manga, current_user)
|
||||
now = db.conn.execute("SELECT datetime('now')").fetchone()[0]
|
||||
db.conn.execute(
|
||||
"UPDATE chapters SET status='pending', pages_done=0, pages_total=0, updated_at=?"
|
||||
" WHERE manga_url=? AND status='failed'", (now, url))
|
||||
db.conn.execute(
|
||||
"UPDATE chapters SET status='pending', pages_done=0, pages_total=0, updated_at=?"
|
||||
" WHERE manga_url=? AND status='done' AND pages_total > 0 AND pages_done < pages_total",
|
||||
(now, url))
|
||||
db.conn.commit()
|
||||
db.reset_failed_chapters(url)
|
||||
return {"ok": True}
|
||||
finally:
|
||||
db.close()
|
||||
@@ -769,7 +743,7 @@ class RenameFolderRequest(BaseModel):
|
||||
folder_name: str
|
||||
@app.post("/api/mangas/rename_folder")
|
||||
async def rename_folder(body: RenameFolderRequest, current_user: dict = Depends(get_current_user)):
|
||||
new_folder = _safe_name(body.folder_name)
|
||||
new_folder = safe_name(body.folder_name)
|
||||
if not new_folder:
|
||||
raise HTTPException(status_code=400, detail="Некорректное имя папки")
|
||||
db = StateDB()
|
||||
@@ -786,21 +760,9 @@ async def rename_folder(body: RenameFolderRequest, current_user: dict = Depends(
|
||||
if new_dir.exists():
|
||||
raise HTTPException(status_code=400, detail=f"Папка '{new_folder}' уже существует")
|
||||
if old_dir.exists():
|
||||
import shutil
|
||||
shutil.move(str(old_dir), str(new_dir))
|
||||
logger.info("Папка переименована: {} → {}", old_dir, new_dir)
|
||||
chapters = db.get_all_chapters(body.url)
|
||||
for ch in chapters:
|
||||
updates = {}
|
||||
for col in ("output_cbz", "output_pdf", "output_epub"):
|
||||
p = ch.get(col)
|
||||
if p and str(old_dir) in p:
|
||||
updates[col] = p.replace(str(old_dir), str(new_dir))
|
||||
if updates:
|
||||
sets = ", ".join(f"{k}=?" for k in updates)
|
||||
db.conn.execute(f"UPDATE chapters SET {sets} WHERE chapter_url=?",
|
||||
[*updates.values(), ch["chapter_url"]])
|
||||
db.conn.commit()
|
||||
db.update_chapter_output_paths(body.url, str(old_dir), str(new_dir))
|
||||
db.set_folder_name(body.url, new_folder)
|
||||
await ws_manager.broadcast({"type": "manga_folder_renamed",
|
||||
"url": body.url, "folder_name": new_folder})
|
||||
@@ -816,11 +778,7 @@ async def force_redownload(url: str, _: dict = Depends(require_admin)):
|
||||
raise HTTPException(status_code=404, detail="Манга не найдена")
|
||||
if manga["status"] == "downloading" and url in active_tasks:
|
||||
raise HTTPException(status_code=400, detail="Сначала остановите загрузку")
|
||||
now = db.conn.execute("SELECT datetime('now')").fetchone()[0]
|
||||
db.conn.execute(
|
||||
"UPDATE chapters SET status='pending', pages_done=0, pages_total=0, updated_at=? WHERE manga_url=?",
|
||||
(now, url))
|
||||
db.conn.commit()
|
||||
db.reset_all_chapters(url)
|
||||
db.update_manga_status(url, "queued")
|
||||
await download_queue.put({"url": url, "fmt": manga["format"], "resume": False})
|
||||
await ws_manager.broadcast({"type": "manga_queued", "url": url, "format": manga["format"]})
|
||||
@@ -876,13 +834,9 @@ async def delete_manga(url: str, delete_files: bool = False, _: dict = Depends(r
|
||||
manga_dir = _manga_folder(manga)
|
||||
if manga_dir.exists() and manga_dir.is_dir():
|
||||
deleted_size = _dir_size(manga_dir)
|
||||
import shutil
|
||||
shutil.rmtree(str(manga_dir))
|
||||
logger.info("Удалена папка: {} ({} байт)", manga_dir, deleted_size)
|
||||
db.conn.execute("DELETE FROM chapters WHERE manga_url=?", (url,))
|
||||
db.conn.execute("DELETE FROM history WHERE manga_url=?", (url,))
|
||||
db.conn.execute("DELETE FROM mangas WHERE url=?", (url,))
|
||||
db.conn.commit()
|
||||
db.delete_manga_cascade(url)
|
||||
return {"ok": True, "deleted_size": deleted_size}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -95,32 +95,6 @@ class BrowserManager:
|
||||
page = await ctx.new_page()
|
||||
return ctx, page
|
||||
|
||||
async def navigate(self, page: Page, url: str, timeout: int = 60_000,
|
||||
referer: str | None = None) -> bool:
|
||||
"""
|
||||
Открывает URL и ждёт загрузки.
|
||||
referer — явно выставляется в заголовке запроса (обход защиты сервера).
|
||||
Возвращает True при успехе.
|
||||
"""
|
||||
# Если referer не передан явно — берём домен из url
|
||||
if referer is None:
|
||||
from urllib.parse import urlparse
|
||||
p = urlparse(url)
|
||||
referer = f"{p.scheme}://{p.netloc}/"
|
||||
try:
|
||||
logger.debug("Навигация: {} (referer={})", url, referer)
|
||||
response = await page.goto(url, wait_until="domcontentloaded",
|
||||
timeout=timeout, referer=referer)
|
||||
if response and response.status >= 400:
|
||||
logger.warning("HTTP {}: {}", response.status, url)
|
||||
return False
|
||||
# Ждём завершения JS
|
||||
await page.wait_for_load_state("networkidle", timeout=timeout)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error("Ошибка навигации {}: {}", url, e)
|
||||
return False
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.start()
|
||||
return self
|
||||
|
||||
111
src/cli.py
111
src/cli.py
@@ -16,9 +16,11 @@ from loguru import logger
|
||||
from tqdm import tqdm
|
||||
|
||||
from .browser import BrowserManager
|
||||
from .scraper import get_manga_info, get_chapter_images_and_download, Chapter
|
||||
from .exporter import export, ExportFormat
|
||||
from .sources import registry, get_source_for_url
|
||||
from .sources.base import Chapter
|
||||
from .exporter import export, ExportFormat, MangaMeta
|
||||
from .state import StateDB
|
||||
from .utils import safe_name, safe_chapter_name
|
||||
|
||||
OUTPUT_DIR = Path("/app/output")
|
||||
STATE_DIR = Path("/app/state")
|
||||
@@ -80,36 +82,41 @@ def download(ctx, url, fmt, chapters, output, resume, force, concurrency):
|
||||
|
||||
async def _download(url, fmt, chapters_filter, output_dir, resume, force, concurrency, verbose):
|
||||
db = StateDB()
|
||||
db.sync_sources(registry)
|
||||
|
||||
source = get_source_for_url(url, db)
|
||||
if source is None:
|
||||
srcs = registry.all_sources()
|
||||
source = srcs[0] if srcs else None
|
||||
if source is None:
|
||||
logger.error("Источник не определён для URL: {}", url)
|
||||
db.close()
|
||||
return
|
||||
|
||||
async with BrowserManager(headless=True) as bm:
|
||||
ctx, page = await bm.new_page()
|
||||
|
||||
# 1. Получаем список глав
|
||||
manga = await get_manga_info(page, url)
|
||||
manga = await source.get_manga_info(page, url)
|
||||
if not manga:
|
||||
logger.error("Не удалось получить информацию о манге")
|
||||
db.close()
|
||||
return
|
||||
|
||||
manga_dir = output_dir / _safe_name(manga.title)
|
||||
manga_dir = output_dir / safe_name(manga.title_ru or manga.title)
|
||||
manga_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 2. Сохраняем все главы в БД
|
||||
for ch in manga.chapters:
|
||||
db.upsert_chapter(url, ch.url, ch.title, ch.number, ch.volume)
|
||||
|
||||
# 3. Фильтрация
|
||||
chapters = _filter_chapters(manga.chapters, chapters_filter)
|
||||
logger.info("Будет скачано глав: {}", len(chapters))
|
||||
|
||||
# 4. Форматы
|
||||
formats: list[ExportFormat] = ["cbz", "pdf", "epub"] if fmt == "all" else [fmt]
|
||||
|
||||
# 5. Скачиваем каждую главу
|
||||
with tqdm(total=len(chapters), desc="Главы", unit="гл") as pbar:
|
||||
for ch in chapters:
|
||||
pbar.set_description(f"Глава {ch.number}: {ch.title[:30]}")
|
||||
|
||||
# Проверяем статус (resume / force)
|
||||
if force:
|
||||
db.reset_chapter(ch.url)
|
||||
elif resume and db.chapter_status(ch.url) == "done":
|
||||
@@ -118,10 +125,10 @@ async def _download(url, fmt, chapters_filter, output_dir, resume, force, concur
|
||||
continue
|
||||
|
||||
await _process_chapter(
|
||||
bm=bm, ctx=ctx, ch=ch,
|
||||
manga_url=url,
|
||||
source=source, ctx=ctx, ch=ch,
|
||||
manga=manga, manga_url=url,
|
||||
manga_dir=manga_dir, formats=formats,
|
||||
concurrency=concurrency, db=db, force=force,
|
||||
db=db, force=force,
|
||||
)
|
||||
pbar.update(1)
|
||||
|
||||
@@ -130,16 +137,14 @@ async def _download(url, fmt, chapters_filter, output_dir, resume, force, concur
|
||||
db.close()
|
||||
|
||||
|
||||
async def _process_chapter(bm, ctx, ch: Chapter, manga_url: str, manga_dir: Path,
|
||||
formats: list, concurrency: int, db: StateDB, force: bool = False):
|
||||
# Новая страница для каждой главы (чистый контекст)
|
||||
async def _process_chapter(source, ctx, ch: Chapter, manga, manga_url: str,
|
||||
manga_dir: Path, formats: list, db: StateDB, force: bool = False):
|
||||
ch_page = await ctx.new_page()
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir)
|
||||
# Открываем главу и скачиваем изображения за один проход
|
||||
image_paths = await get_chapter_images_and_download(
|
||||
image_paths = await source.get_chapter_images_and_download(
|
||||
ch_page, ch.url, dest_dir=tmp_path, manga_url=manga_url
|
||||
)
|
||||
|
||||
@@ -148,16 +153,27 @@ async def _process_chapter(bm, ctx, ch: Chapter, manga_url: str, manga_dir: Path
|
||||
db.mark_failed(ch.url)
|
||||
return
|
||||
|
||||
ch_name = _safe_chapter_name(ch)
|
||||
ch_name = safe_chapter_name(ch)
|
||||
ch_meta = MangaMeta(
|
||||
series=manga.title_ru or manga.title,
|
||||
series_full=manga.title_full or "",
|
||||
chapter_title=ch.title,
|
||||
number=ch.number,
|
||||
volume=ch.volume,
|
||||
chapters_total=len(manga.chapters),
|
||||
pub_status=manga.pub_status,
|
||||
source_url=manga_url,
|
||||
summary=manga.description,
|
||||
genre=", ".join(manga.genres) if manga.genres else "",
|
||||
)
|
||||
|
||||
for fmt in formats:
|
||||
out_file = manga_dir / f"{ch_name}.{fmt}"
|
||||
# При --force удаляем старый файл перед перезаписью
|
||||
if force and out_file.exists():
|
||||
out_file.unlink()
|
||||
logger.debug("Удалён старый файл: {}", out_file.name)
|
||||
try:
|
||||
export(image_paths, out_file, fmt, manga_dir.name, ch.title)
|
||||
export(image_paths, out_file, fmt, meta=ch_meta)
|
||||
db.mark_done(ch.url, fmt, str(out_file))
|
||||
except Exception as e:
|
||||
logger.error("Ошибка экспорта {}: {}", fmt, e)
|
||||
@@ -180,15 +196,28 @@ def analyze(ctx, url):
|
||||
|
||||
|
||||
async def _analyze(url: str):
|
||||
db = StateDB()
|
||||
db.sync_sources(registry)
|
||||
|
||||
source = get_source_for_url(url, db)
|
||||
if source is None:
|
||||
srcs = registry.all_sources()
|
||||
source = srcs[0] if srcs else None
|
||||
if source is None:
|
||||
click.echo("❌ Источник не найден")
|
||||
db.close()
|
||||
return
|
||||
|
||||
async with BrowserManager(headless=True) as bm:
|
||||
_, page = await bm.new_page()
|
||||
manga = await get_manga_info(page, url)
|
||||
manga = await source.get_manga_info(page, url)
|
||||
|
||||
if not manga:
|
||||
click.echo("❌ Не удалось получить информацию")
|
||||
db.close()
|
||||
return
|
||||
|
||||
click.echo(f"\n📚 Манга: {manga.title}")
|
||||
click.echo(f"\n📚 Манга: {manga.title_ru or manga.title}")
|
||||
click.echo(f"🔗 URL: {manga.url}")
|
||||
click.echo(f"📖 Глав: {len(manga.chapters)}\n")
|
||||
|
||||
@@ -198,64 +227,34 @@ async def _analyze(url: str):
|
||||
if len(manga.chapters) > 20:
|
||||
click.echo(f" ... и ещё {len(manga.chapters) - 20} глав")
|
||||
|
||||
# Проверяем одну главу
|
||||
if manga.chapters:
|
||||
first = manga.chapters[-1]
|
||||
click.echo(f"\n🔍 Проверяем первую главу: {first.url}")
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
paths = await get_chapter_images_and_download(
|
||||
paths = await source.get_chapter_images_and_download(
|
||||
page, first.url, dest_dir=Path(tmp), manga_url=url
|
||||
)
|
||||
click.echo(f" Скачано изображений: {len(paths)}")
|
||||
for p in paths[:3]:
|
||||
click.echo(f" {p.name} ({p.stat().st_size} байт)")
|
||||
|
||||
db.close()
|
||||
|
||||
|
||||
# ── Утилиты ───────────────────────────────────
|
||||
|
||||
def _safe_name(s: str) -> str:
|
||||
return re.sub(r'[^\w\s\-]', '', s).strip().replace(" ", "_")[:80]
|
||||
|
||||
|
||||
def _safe_chapter_name(ch: Chapter) -> str:
|
||||
vol = f"v{ch.volume:02d}_" if ch.volume else ""
|
||||
return f"{vol}ch{ch.number:06.1f}"
|
||||
|
||||
|
||||
def _filter_chapters(chapters: list[Chapter], filter_str: str | None) -> list[Chapter]:
|
||||
if not filter_str:
|
||||
return chapters
|
||||
|
||||
# "1-10" → диапазон
|
||||
m = re.match(r"^(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$", filter_str)
|
||||
if m:
|
||||
lo, hi = float(m.group(1)), float(m.group(2))
|
||||
return [c for c in chapters if lo <= c.number <= hi]
|
||||
|
||||
# "1,3,7" → список
|
||||
nums = {float(x.strip()) for x in filter_str.split(",")}
|
||||
return [c for c in chapters if c.number in nums]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -131,8 +131,12 @@ def _export_pdf(images: list[Path], out: Path, meta: MangaMeta):
|
||||
def _export_pdf_pillow(images: list[Path], out: Path):
|
||||
from PIL import Image
|
||||
pil_images = [Image.open(p).convert("RGB") for p in images]
|
||||
if pil_images:
|
||||
pil_images[0].save(out, save_all=True, append_images=pil_images[1:], format="PDF")
|
||||
try:
|
||||
if pil_images:
|
||||
pil_images[0].save(out, save_all=True, append_images=pil_images[1:], format="PDF")
|
||||
finally:
|
||||
for img in pil_images:
|
||||
img.close()
|
||||
|
||||
|
||||
def _patch_pdf_meta(pdf_path: Path, meta: MangaMeta):
|
||||
|
||||
@@ -7,6 +7,7 @@ import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from loguru import logger
|
||||
from playwright.async_api import Page
|
||||
@@ -88,7 +89,6 @@ class ReadmangaSource:
|
||||
ch_id = chapter_url.split("/")[-1]
|
||||
logger.info("[{}] Загружаем главу: {}", ch_id, chapter_url)
|
||||
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(chapter_url)
|
||||
parts = parsed.path.strip("/").split("/")
|
||||
manga_slug = parts[0] if parts else ""
|
||||
@@ -277,8 +277,6 @@ class ReadmangaSource:
|
||||
elapsed = time.monotonic() - t_start
|
||||
logger.info("[{}] Перехвачено: {}/{} за {:.1f}с", ch_id, done, total, elapsed)
|
||||
|
||||
filename_to_idx = {_base(u).split("/")[-1]: i for i, u in enumerate(image_urls)}
|
||||
|
||||
paths: dict[int, Path] = {}
|
||||
unmatched_other: list[str] = []
|
||||
for base_url, body in captured.items():
|
||||
@@ -337,7 +335,6 @@ class ReadmangaSource:
|
||||
|
||||
async def _navigate(page: Page, url: str, retries: int = 3,
|
||||
referer: str | None = None) -> bool:
|
||||
from urllib.parse import urlparse
|
||||
if referer is None:
|
||||
p = urlparse(url)
|
||||
referer = f"{p.scheme}://{p.netloc}/"
|
||||
|
||||
64
src/state.py
64
src/state.py
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
@@ -402,10 +402,6 @@ class StateDB:
|
||||
self.conn.commit()
|
||||
return count
|
||||
|
||||
def increment_manga_chapters_done(self, url: str):
|
||||
# Оставлен для совместимости, но не используется в воркере
|
||||
pass
|
||||
|
||||
def get_manga(self, url: str) -> Optional[dict]:
|
||||
cur = self.conn.execute("""
|
||||
SELECT m.*, u.username AS added_by_username
|
||||
@@ -428,6 +424,57 @@ class StateDB:
|
||||
row = cur.fetchone()
|
||||
return row["format"] if row else "cbz"
|
||||
|
||||
def get_chapter_stats(self, manga_url: str) -> dict:
|
||||
"""Returns done/failed/partial chapter counts in a single query."""
|
||||
row = self.conn.execute("""
|
||||
SELECT
|
||||
COUNT(CASE WHEN status='done' THEN 1 END) as done,
|
||||
COUNT(CASE WHEN status='failed' THEN 1 END) as failed,
|
||||
COUNT(CASE WHEN status='done' AND pages_total > 0
|
||||
AND pages_done < pages_total THEN 1 END) as partial
|
||||
FROM chapters WHERE manga_url=?
|
||||
""", (manga_url,)).fetchone()
|
||||
return {"done": row[0], "failed": row[1], "partial": row[2]}
|
||||
|
||||
def reset_all_chapters(self, manga_url: str) -> None:
|
||||
"""Resets ALL chapters to pending (used by force-redownload)."""
|
||||
self.conn.execute(
|
||||
"UPDATE chapters SET status='pending', pages_done=0, pages_total=0, updated_at=?"
|
||||
" WHERE manga_url=?",
|
||||
(_now(), manga_url)
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def delete_manga_cascade(self, manga_url: str) -> None:
|
||||
"""Deletes manga and all related chapters and history."""
|
||||
self.conn.execute("DELETE FROM chapters WHERE manga_url=?", (manga_url,))
|
||||
self.conn.execute("DELETE FROM history WHERE manga_url=?", (manga_url,))
|
||||
self.conn.execute("DELETE FROM mangas WHERE url=?", (manga_url,))
|
||||
self.conn.commit()
|
||||
|
||||
def update_chapter_output_paths(self, manga_url: str, old_prefix: str, new_prefix: str) -> None:
|
||||
"""Replaces old_prefix with new_prefix in chapter output paths after folder rename."""
|
||||
chapters = self.get_all_chapters(manga_url)
|
||||
for ch in chapters:
|
||||
for col in ("output_cbz", "output_pdf", "output_epub"):
|
||||
p = ch.get(col)
|
||||
if p and old_prefix in p:
|
||||
self.conn.execute(
|
||||
f"UPDATE chapters SET {col}=?, updated_at=? WHERE chapter_url=?",
|
||||
(p.replace(old_prefix, new_prefix), _now(), ch["chapter_url"])
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def get_news(self, limit: int = 100) -> list[dict]:
|
||||
"""Returns recently downloaded chapters for the news feed."""
|
||||
cur = self.conn.execute("""
|
||||
SELECT h.*, m.title as manga_title, m.title_ru
|
||||
FROM history h LEFT JOIN mangas m ON h.manga_url = m.url
|
||||
WHERE h.event_type IN ('downloaded', 'auto_downloaded')
|
||||
ORDER BY h.created_at DESC LIMIT ?
|
||||
""", (limit,))
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
|
||||
# ── Chapters ──────────────────────────────────
|
||||
|
||||
def upsert_chapter(self, manga_url: str, chapter_url: str,
|
||||
@@ -451,6 +498,8 @@ class StateDB:
|
||||
self.conn.commit()
|
||||
|
||||
def mark_done(self, chapter_url: str, fmt: str, output_path: str):
|
||||
if fmt not in _ALLOWED_FMTS:
|
||||
raise ValueError(f"Unknown format: {fmt}")
|
||||
col = f"output_{fmt}"
|
||||
self.conn.execute(f"""
|
||||
UPDATE chapters SET status='done', {col}=?, updated_at=?
|
||||
@@ -624,8 +673,11 @@ class StateDB:
|
||||
self.conn.close()
|
||||
|
||||
|
||||
_ALLOWED_FMTS = frozenset({"cbz", "pdf", "epub"})
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.utcnow().isoformat()
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None).isoformat()
|
||||
|
||||
|
||||
def _extract_domain(url: str) -> str:
|
||||
|
||||
15
src/utils.py
Normal file
15
src/utils.py
Normal file
@@ -0,0 +1,15 @@
|
||||
"""
|
||||
Общие утилиты, используемые в нескольких модулях.
|
||||
"""
|
||||
import re
|
||||
|
||||
from .sources.base import Chapter
|
||||
|
||||
|
||||
def safe_name(s: str) -> str:
|
||||
return re.sub(r'[^\w\s\-]', '', s).strip().replace(" ", "_")[:80]
|
||||
|
||||
|
||||
def safe_chapter_name(ch: Chapter) -> str:
|
||||
vol = f"v{ch.volume:02d}_" if ch.volume else ""
|
||||
return f"{vol}ch{ch.number:06.1f}"
|
||||
@@ -3,7 +3,6 @@
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
@@ -13,9 +12,9 @@ from loguru import logger
|
||||
from .browser import BrowserManager
|
||||
from .sources import registry, get_source_for_url, extract_domain
|
||||
from .sources.base import Chapter, MangaInfo
|
||||
from .scraper import get_manga_info, get_chapter_images_and_download # shim для обратной совместимости
|
||||
from .exporter import export, MangaMeta
|
||||
from .state import StateDB
|
||||
from .utils import safe_name, safe_chapter_name
|
||||
|
||||
OUTPUT_DIR = Path("/app/output")
|
||||
|
||||
@@ -23,15 +22,6 @@ OUTPUT_DIR = Path("/app/output")
|
||||
CHAPTER_CONCURRENCY = int(os.getenv("CHAPTER_CONCURRENCY", "3"))
|
||||
|
||||
|
||||
def _safe_name(s: str) -> str:
|
||||
return re.sub(r'[^\w\s\-]', '', s).strip().replace(" ", "_")[:80]
|
||||
|
||||
|
||||
def _safe_chapter_name(ch: Chapter) -> str:
|
||||
vol = f"v{ch.volume:02d}_" if ch.volume else ""
|
||||
return f"{vol}ch{ch.number:06.1f}"
|
||||
|
||||
|
||||
async def download_manga(
|
||||
url: str,
|
||||
fmt: str = "cbz",
|
||||
@@ -111,7 +101,7 @@ async def download_manga(
|
||||
_db_manga = await db_call(db.get_manga, url)
|
||||
folder_name = (
|
||||
(_db_manga.get("folder_name") if _db_manga else None)
|
||||
or _safe_name(manga.title_ru or manga.title)
|
||||
or safe_name(manga.title_ru or manga.title)
|
||||
)
|
||||
manga_dir = output_dir / folder_name
|
||||
manga_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -193,18 +183,19 @@ async def download_manga(
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir)
|
||||
pages_done_count = [0]
|
||||
pages_done = 0
|
||||
|
||||
async def on_page(page_idx: int, pages_total: int):
|
||||
pages_done_count[0] += 1
|
||||
nonlocal pages_done
|
||||
pages_done += 1
|
||||
await db_call(db.update_chapter_pages,
|
||||
ch.url, pages_total, pages_done_count[0])
|
||||
ch.url, pages_total, pages_done)
|
||||
await emit({
|
||||
"type": "page_done",
|
||||
"url": url,
|
||||
"chapter_url": ch.url,
|
||||
"page_idx": page_idx,
|
||||
"pages_done": pages_done_count[0],
|
||||
"pages_done": pages_done,
|
||||
"pages_total": pages_total,
|
||||
})
|
||||
|
||||
@@ -226,7 +217,7 @@ async def download_manga(
|
||||
"chapter_url": ch.url})
|
||||
return
|
||||
|
||||
ch_name = _safe_chapter_name(ch)
|
||||
ch_name = safe_chapter_name(ch)
|
||||
ch_meta = MangaMeta(
|
||||
series=manga.title_ru or manga.title,
|
||||
series_full=manga.title_full or "",
|
||||
|
||||
Reference in New Issue
Block a user