This commit is contained in:
2026-05-01 03:50:25 +03:00
parent 43597be020
commit bc7b5bfe37
10 changed files with 549 additions and 185 deletions

View File

@@ -16,9 +16,11 @@ from loguru import logger
from tqdm import tqdm
from .browser import BrowserManager
from .scraper import get_manga_info, get_chapter_images_and_download, Chapter
from .exporter import export, ExportFormat
from .sources import registry, get_source_for_url
from .sources.base import Chapter
from .exporter import export, ExportFormat, MangaMeta
from .state import StateDB
from .utils import safe_name, safe_chapter_name
OUTPUT_DIR = Path("/app/output")
STATE_DIR = Path("/app/state")
@@ -80,36 +82,41 @@ def download(ctx, url, fmt, chapters, output, resume, force, concurrency):
async def _download(url, fmt, chapters_filter, output_dir, resume, force, concurrency, verbose):
db = StateDB()
db.sync_sources(registry)
source = get_source_for_url(url, db)
if source is None:
srcs = registry.all_sources()
source = srcs[0] if srcs else None
if source is None:
logger.error("Источник не определён для URL: {}", url)
db.close()
return
async with BrowserManager(headless=True) as bm:
ctx, page = await bm.new_page()
# 1. Получаем список глав
manga = await get_manga_info(page, url)
manga = await source.get_manga_info(page, url)
if not manga:
logger.error("Не удалось получить информацию о манге")
db.close()
return
manga_dir = output_dir / _safe_name(manga.title)
manga_dir = output_dir / safe_name(manga.title_ru or manga.title)
manga_dir.mkdir(parents=True, exist_ok=True)
# 2. Сохраняем все главы в БД
for ch in manga.chapters:
db.upsert_chapter(url, ch.url, ch.title, ch.number, ch.volume)
# 3. Фильтрация
chapters = _filter_chapters(manga.chapters, chapters_filter)
logger.info("Будет скачано глав: {}", len(chapters))
# 4. Форматы
formats: list[ExportFormat] = ["cbz", "pdf", "epub"] if fmt == "all" else [fmt]
# 5. Скачиваем каждую главу
with tqdm(total=len(chapters), desc="Главы", unit="гл") as pbar:
for ch in chapters:
pbar.set_description(f"Глава {ch.number}: {ch.title[:30]}")
# Проверяем статус (resume / force)
if force:
db.reset_chapter(ch.url)
elif resume and db.chapter_status(ch.url) == "done":
@@ -118,10 +125,10 @@ async def _download(url, fmt, chapters_filter, output_dir, resume, force, concur
continue
await _process_chapter(
bm=bm, ctx=ctx, ch=ch,
manga_url=url,
source=source, ctx=ctx, ch=ch,
manga=manga, manga_url=url,
manga_dir=manga_dir, formats=formats,
concurrency=concurrency, db=db, force=force,
db=db, force=force,
)
pbar.update(1)
@@ -130,16 +137,14 @@ async def _download(url, fmt, chapters_filter, output_dir, resume, force, concur
db.close()
async def _process_chapter(bm, ctx, ch: Chapter, manga_url: str, manga_dir: Path,
formats: list, concurrency: int, db: StateDB, force: bool = False):
# Новая страница для каждой главы (чистый контекст)
async def _process_chapter(source, ctx, ch: Chapter, manga, manga_url: str,
manga_dir: Path, formats: list, db: StateDB, force: bool = False):
ch_page = await ctx.new_page()
try:
with tempfile.TemporaryDirectory() as tmpdir:
tmp_path = Path(tmpdir)
# Открываем главу и скачиваем изображения за один проход
image_paths = await get_chapter_images_and_download(
image_paths = await source.get_chapter_images_and_download(
ch_page, ch.url, dest_dir=tmp_path, manga_url=manga_url
)
@@ -148,16 +153,27 @@ async def _process_chapter(bm, ctx, ch: Chapter, manga_url: str, manga_dir: Path
db.mark_failed(ch.url)
return
ch_name = _safe_chapter_name(ch)
ch_name = safe_chapter_name(ch)
ch_meta = MangaMeta(
series=manga.title_ru or manga.title,
series_full=manga.title_full or "",
chapter_title=ch.title,
number=ch.number,
volume=ch.volume,
chapters_total=len(manga.chapters),
pub_status=manga.pub_status,
source_url=manga_url,
summary=manga.description,
genre=", ".join(manga.genres) if manga.genres else "",
)
for fmt in formats:
out_file = manga_dir / f"{ch_name}.{fmt}"
# При --force удаляем старый файл перед перезаписью
if force and out_file.exists():
out_file.unlink()
logger.debug("Удалён старый файл: {}", out_file.name)
try:
export(image_paths, out_file, fmt, manga_dir.name, ch.title)
export(image_paths, out_file, fmt, meta=ch_meta)
db.mark_done(ch.url, fmt, str(out_file))
except Exception as e:
logger.error("Ошибка экспорта {}: {}", fmt, e)
@@ -180,15 +196,28 @@ def analyze(ctx, url):
async def _analyze(url: str):
db = StateDB()
db.sync_sources(registry)
source = get_source_for_url(url, db)
if source is None:
srcs = registry.all_sources()
source = srcs[0] if srcs else None
if source is None:
click.echo("❌ Источник не найден")
db.close()
return
async with BrowserManager(headless=True) as bm:
_, page = await bm.new_page()
manga = await get_manga_info(page, url)
manga = await source.get_manga_info(page, url)
if not manga:
click.echo("Не удалось получить информацию")
db.close()
return
click.echo(f"\n📚 Манга: {manga.title}")
click.echo(f"\n📚 Манга: {manga.title_ru or manga.title}")
click.echo(f"🔗 URL: {manga.url}")
click.echo(f"📖 Глав: {len(manga.chapters)}\n")
@@ -198,64 +227,34 @@ async def _analyze(url: str):
if len(manga.chapters) > 20:
click.echo(f" ... и ещё {len(manga.chapters) - 20} глав")
# Проверяем одну главу
if manga.chapters:
first = manga.chapters[-1]
click.echo(f"\n🔍 Проверяем первую главу: {first.url}")
import tempfile
with tempfile.TemporaryDirectory() as tmp:
paths = await get_chapter_images_and_download(
paths = await source.get_chapter_images_and_download(
page, first.url, dest_dir=Path(tmp), manga_url=url
)
click.echo(f" Скачано изображений: {len(paths)}")
for p in paths[:3]:
click.echo(f" {p.name} ({p.stat().st_size} байт)")
db.close()
# ── Утилиты ───────────────────────────────────
def _safe_name(s: str) -> str:
return re.sub(r'[^\w\s\-]', '', s).strip().replace(" ", "_")[:80]
def _safe_chapter_name(ch: Chapter) -> str:
vol = f"v{ch.volume:02d}_" if ch.volume else ""
return f"{vol}ch{ch.number:06.1f}"
def _filter_chapters(chapters: list[Chapter], filter_str: str | None) -> list[Chapter]:
if not filter_str:
return chapters
# "1-10" → диапазон
m = re.match(r"^(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$", filter_str)
if m:
lo, hi = float(m.group(1)), float(m.group(2))
return [c for c in chapters if lo <= c.number <= hi]
# "1,3,7" → список
nums = {float(x.strip()) for x in filter_str.split(",")}
return [c for c in chapters if c.number in nums]
if __name__ == "__main__":
cli()