feat: switch WEB to shared hot pool with autoscale

2026-04-14 13:37:45 +00:00
parent 6095d53854
commit 605b269f74
2 changed files with 256 additions and 24 deletions
--- a/app/main.py
+++ b/app/main.py
@@ -44,6 +44,8 @@ LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
 TRAEFIK_INTERNAL_URL = os.getenv("TRAEFIK_INTERNAL_URL", "http://traefik")
 PREWARM_POOL_SIZE = int(os.getenv("PREWARM_POOL_SIZE", "0"))
 UNIVERSAL_POOL_SIZE = int(os.getenv("UNIVERSAL_POOL_SIZE", "5"))
+WEB_POOL_SIZE = int(os.getenv("WEB_POOL_SIZE", "5"))
+WEB_POOL_BUFFER = int(os.getenv("WEB_POOL_BUFFER", "2"))
 ICON_UPLOAD_MAX_BYTES = 2 * 1024 * 1024
 ICON_UPLOAD_TYPES = {
    "image/png": "png",
@@ -223,6 +225,10 @@ def universal_container_name(slot: int) -> str:
    return f"portal-universal-{slot}"


+def web_pool_container_name(slot: int) -> str:
+    return f"portal-webpool-{slot}"
+
+
 def ensure_icons_dir() -> None:
    SERVICE_ICONS_DIR.mkdir(parents=True, exist_ok=True)

@@ -422,6 +428,67 @@ def ensure_universal_pool() -> None:
        logger.info("universal_pool_container_started slot=%s", i)


+def ensure_web_pool(target_size: Optional[int] = None) -> None:
+    desired = max(0, WEB_POOL_SIZE if target_size is None else target_size)
+    d = docker_client()
+    image = "portal-universal-runtime:latest"
+
+    for i in range(desired, 100):
+        name = web_pool_container_name(i)
+        try:
+            c = d.containers.get(name)
+            c.stop(timeout=5)
+        except docker.errors.NotFound:
+            break
+        except Exception:
+            logger.exception("web_pool_scale_down_failed slot=%s", i)
+
+    for i in range(desired):
+        name = web_pool_container_name(i)
+        path = f"/w/{i}/"
+        router = f"wpool-{i}"
+        labels = {
+            "traefik.enable": "true",
+            "traefik.docker.network": "portal_net",
+            f"traefik.http.routers.{router}.rule": f"PathPrefix(`{path}`)",
+            f"traefik.http.routers.{router}.entrypoints": "websecure",
+            f"traefik.http.routers.{router}.tls": "true",
+            f"traefik.http.routers.{router}.priority": "9450",
+            f"traefik.http.routers.{router}.middlewares": f"{router}-strip",
+            f"traefik.http.middlewares.{router}-strip.stripprefix.prefixes": path[:-1],
+            f"traefik.http.services.{router}.loadbalancer.server.port": "6080",
+            "portal.pool": "1",
+            "portal.pool.kind": "web",
+            "portal.pool.slot": str(i),
+        }
+        env = {
+            "IDLE_TIMEOUT": str(SESSION_IDLE_SECONDS),
+            "ENABLE_HEARTBEAT": "0",
+            "SESSION_ID": f"webpool-{i}",
+        }
+        try:
+            c = d.containers.get(name)
+            if c.status != "running":
+                c.start()
+            continue
+        except docker.errors.NotFound:
+            pass
+        except Exception:
+            logger.exception("web_pool_check_failed slot=%s", i)
+            continue
+
+        d.containers.run(
+            image=image,
+            name=name,
+            detach=True,
+            auto_remove=True,
+            network="portal_net",
+            labels=labels,
+            environment=env,
+        )
+        logger.info("web_pool_container_started slot=%s", i)
+
+
 def get_universal_pool_status() -> dict:
    desired = max(0, UNIVERSAL_POOL_SIZE)
    if desired <= 0:
@@ -445,6 +512,29 @@ def get_universal_pool_status() -> dict:
    }


+def get_web_pool_status() -> dict:
+    desired = max(0, WEB_POOL_SIZE)
+    if desired <= 0:
+        return {"desired": 0, "running": 0, "total": 0, "health": "down", "names": []}
+    d = docker_client()
+    names = [web_pool_container_name(i) for i in range(desired)]
+    containers = []
+    for name in names:
+        try:
+            containers.append(d.containers.get(name))
+        except Exception:
+            continue
+    running = sum(1 for c in containers if c.status == "running")
+    health = "ok" if running >= min(desired, 1) else "down"
+    return {
+        "desired": desired,
+        "running": running,
+        "total": len(containers),
+        "names": sorted(c.name for c in containers),
+        "health": health,
+    }
+
+
 def acquire_universal_slot(db: Session) -> int:
    cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS)
    q = select(SessionModel).where(
@@ -473,6 +563,33 @@ def acquire_universal_slot(db: Session) -> int:
    return 0


+def acquire_web_pool_slot(db: Session) -> int:
+    cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS)
+    q = select(SessionModel).where(
+        SessionModel.status == SessionStatus.ACTIVE,
+        SessionModel.container_id.like("WEBPOOLIDX:%"),
+        SessionModel.last_access_at >= cutoff,
+    )
+    active = db.scalars(q).all()
+    busy = set()
+    for sess in active:
+        try:
+            busy.add(int((sess.container_id or "").split(":", 1)[1]))
+        except Exception:
+            continue
+
+    # Keep headroom: when active sessions are close to hot pool capacity,
+    # proactively warm up extra slots.
+    auto_target = max(WEB_POOL_SIZE, len(active) + max(0, WEB_POOL_BUFFER))
+    if auto_target > WEB_POOL_SIZE:
+        ensure_web_pool(auto_target)
+
+    for i in range(max(0, auto_target)):
+        if i not in busy:
+            return i
+    return 0
+
+
 def dispatch_universal_target(slot: int, service: Service) -> None:
    name = universal_container_name(slot)
    url = ""
@@ -507,6 +624,23 @@ def dispatch_universal_target(slot: int, service: Service) -> None:
        raise last_exc


+def dispatch_web_pool_target(slot: int, service: Service) -> None:
+    name = web_pool_container_name(slot)
+    target_url = normalize_web_target(service.target)
+    url = f"http://{name}:7000/open"
+    last_exc = None
+    for _ in range(8):
+        try:
+            resp = requests.post(url, json={"url": target_url}, timeout=3)
+            resp.raise_for_status()
+            return
+        except Exception as exc:
+            last_exc = exc
+            time.sleep(0.4)
+    if last_exc:
+        raise last_exc
+
+
 def create_runtime_container(service: Service, session_id: str):
    d = docker_client()
    router = session_router_name(session_id)
@@ -706,7 +840,11 @@ def route_ready(path: str) -> bool:
 def container_running(container_id: Optional[str]) -> bool:
    if not container_id:
        return False
-    if container_id.startswith("POOL:") or container_id.startswith("POOLIDX:"):
+    if (
+        container_id.startswith("POOL:")
+        or container_id.startswith("POOLIDX:")
+        or container_id.startswith("WEBPOOLIDX:")
+    ):
        return True
    try:
        c = docker_client().containers.get(container_id)
@@ -798,6 +936,8 @@ def get_warm_containers_for_service(service: Service) -> list:


 def get_pool_status_for_service(service: Service) -> dict:
+    if service.type == ServiceType.WEB:
+        return get_web_pool_status()
    if service_uses_universal_pool(service):
        return get_universal_pool_status()
    desired = desired_pool_size(service)
@@ -822,6 +962,39 @@ def get_pool_status_for_service(service: Service) -> dict:


 def get_pool_detailed_status(service: Service) -> dict:
+    if service.type == ServiceType.WEB:
+        d = docker_client()
+        pool = get_web_pool_status()
+        details = []
+        for i in range(max(0, pool["desired"])):
+            name = web_pool_container_name(i)
+            try:
+                c = d.containers.get(name)
+            except Exception:
+                continue
+            attrs = c.attrs or {}
+            state = (attrs.get("State") or {}).get("Status", c.status)
+            details.append(
+                {
+                    "name": c.name,
+                    "status": c.status,
+                    "state": state,
+                    "created": attrs.get("Created", ""),
+                    "image": c.image.tags[0] if c.image.tags else "",
+                    "labels_ok": True,
+                }
+            )
+        return {
+            "service_id": service.id,
+            "slug": service.slug,
+            "type": service.type.value,
+            "desired": pool["desired"],
+            "running": pool["running"],
+            "total": pool["total"],
+            "health": pool["health"],
+            "containers": details,
+            "updated_at": now_utc().isoformat(),
+        }
    if service_uses_universal_pool(service):
        d = docker_client()
        pool = get_universal_pool_status()
@@ -939,12 +1112,15 @@ def cleanup_loop():
        db = SessionLocal()
        try:
            ensure_universal_pool()
+            ensure_web_pool()
            for svc in db.scalars(
                select(Service).where(
                    Service.active == True,
                    Service.type.in_([ServiceType.WEB, ServiceType.RDP]),
                )
            ).all():
+                if svc.type == ServiceType.WEB:
+                    continue
                if not service_uses_universal_pool(svc):
                    ensure_warm_pool(svc)
            cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS)
@@ -954,7 +1130,11 @@ def cleanup_loop():
            )
            stale = db.scalars(q).all()
            for sess in stale:
-                if sess.container_id and not (sess.container_id.startswith("POOL:") or sess.container_id.startswith("POOLIDX:")):
+                if sess.container_id and not (
+                    sess.container_id.startswith("POOL:")
+                    or sess.container_id.startswith("POOLIDX:")
+                    or sess.container_id.startswith("WEBPOOLIDX:")
+                ):
                    stop_runtime_container(sess.container_id)
                sess.status = SessionStatus.EXPIRED
            if stale:
@@ -998,12 +1178,15 @@ def startup_event():
    db = SessionLocal()
    try:
        ensure_universal_pool()
+        ensure_web_pool()
        for svc in db.scalars(
            select(Service).where(
                Service.active == True,
                Service.type.in_([ServiceType.WEB, ServiceType.RDP]),
            )
        ).all():
+            if svc.type == ServiceType.WEB:
+                continue
            if not service_uses_universal_pool(svc):
                ensure_warm_pool(svc)
    finally:
@@ -1057,10 +1240,11 @@ def admin_page(request: Request, admin: User = Depends(require_admin), db: Sessi
            "desired": st["desired"],
            "active_sessions": get_active_sessions_count(db, sid),
        }
+    web_pool = get_web_pool_status()
    web_totals = {
        "services": len(web_services),
-        "running": sum(service_health[s.id]["running"] for s in web_services),
-        "desired": sum(service_health[s.id]["desired"] for s in web_services),
+        "running": web_pool["running"],
+        "desired": web_pool["desired"],
        "active_sessions": sum(service_health[s.id]["active_sessions"] for s in web_services),
    }
    recent_sessions = db.execute(
@@ -1104,6 +1288,8 @@ def admin_page(request: Request, admin: User = Depends(require_admin), db: Sessi
            "pool_status": pool_status,
            "service_health": service_health,
            "web_totals": web_totals,
+            "web_pool_size": WEB_POOL_SIZE,
+            "web_pool_buffer": WEB_POOL_BUFFER,
            "recent_sessions": recent_sessions,
            "open_stats": open_stats,
            "csrf_token": request.cookies.get(CSRF_COOKIE, ""),
@@ -1153,6 +1339,29 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
        raise HTTPException(status_code=403, detail="ACL denied")

    session_id = str(uuid.uuid4())
+    if service.type == ServiceType.WEB and WEB_POOL_SIZE > 0:
+        try:
+            ensure_web_pool()
+            slot = acquire_web_pool_slot(db)
+            dispatch_web_pool_target(slot, service)
+        except Exception as exc:
+            logger.exception("web_pool_dispatch_failed slug=%s user_id=%s", slug, user.id)
+            audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
+            raise HTTPException(status_code=502, detail="WEB runtime failed to switch target")
+        session_obj = SessionModel(
+            id=session_id,
+            user_id=user.id,
+            service_id=service.id,
+            container_id=f"WEBPOOLIDX:{slot}",
+            status=SessionStatus.ACTIVE,
+            created_at=now_utc(),
+            last_access_at=now_utc(),
+        )
+        db.add(session_obj)
+        db.commit()
+        audit(db, "SESSION_CREATE_WEB_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id)
+        return RedirectResponse(url=f"/w/{slot}/?sid={session_id}", status_code=303)
+
    if service_uses_universal_pool(service):
        try:
            ensure_universal_pool()
@@ -1406,7 +1615,15 @@ def session_status(session_id: str, user: User = Depends(require_user), db: Sess
        raise HTTPException(status_code=410, detail="Session is not active")
    service = db.get(Service, sess.service_id)
    pooled_web = bool(sess.container_id and sess.container_id.startswith("POOL:") and service and service.type == ServiceType.WEB)
+    web_pool_idx = None
+    if sess.container_id and sess.container_id.startswith("WEBPOOLIDX:"):
+        try:
+            web_pool_idx = int(sess.container_id.split(":", 1)[1])
+        except Exception:
+            web_pool_idx = None
    route_path = f"/svc/{service.slug}/" if pooled_web and service else f"/s/{session_id}/"
+    if web_pool_idx is not None:
+        route_path = f"/w/{web_pool_idx}/"
    route_ok = route_ready(route_path)
    running = container_running(sess.container_id)
    ready = running and route_ok
@@ -1421,6 +1638,8 @@ def session_status(session_id: str, user: User = Depends(require_user), db: Sess
    }
    if pooled_web:
        payload["redirect_url"] = f"/s/{session_id}/view"
+    if web_pool_idx is not None:
+        payload["redirect_url"] = f"/w/{web_pool_idx}/?sid={session_id}"
    return payload


@@ -1538,6 +1757,9 @@ def prewarm_now(service_id: int, request: Request, _: User = Depends(require_adm
    service = db.get(Service, service_id)
    if not service:
        raise HTTPException(status_code=404, detail="Service not found")
+    if service.type == ServiceType.WEB:
+        ensure_web_pool()
+        return {"ok": True, "pool": get_web_pool_status()}
    if service_uses_universal_pool(service):
        ensure_universal_pool()
        return {"ok": True, "pool": get_universal_pool_status()}
@@ -1545,6 +1767,16 @@ def prewarm_now(service_id: int, request: Request, _: User = Depends(require_adm
    return {"ok": True, "pool": get_pool_status_for_service(service)}


+@app.put("/api/admin/web-pool-size")
+def update_web_pool_size(payload: dict, request: Request, _: User = Depends(require_admin)):
+    validate_csrf(request)
+    global WEB_POOL_SIZE
+    value = max(0, int(payload.get("size", WEB_POOL_SIZE)))
+    WEB_POOL_SIZE = value
+    ensure_web_pool()
+    return {"ok": True, "size": WEB_POOL_SIZE, "pool": get_web_pool_status()}
+
+
@app.post("/api/admin/users")
 def create_user(payload: dict, request: Request, _: User = Depends(require_admin), db: Session = Depends(get_db)):
    validate_csrf(request)
--- a/app/templates/admin.html
+++ b/app/templates/admin.html
@@ -19,7 +19,7 @@
    <section class="panel">
      <div class="admin-intro">
        Основной режим: <b>WEB</b>. Пользователь выбирает сервис, а портал открывает нужный URL в заранее прогретом браузере.
-        Поле <b>pool size</b> задаёт, сколько таких прогретых контейнеров держать для конкретного сервиса.
+        Для WEB используется <b>общий пул</b> горячих контейнеров с автодоращиванием по занятости.
      </div>
    </section>
    <section class="panel">
@@ -90,7 +90,15 @@
    <section id="tab-web" class="panel admin-tab" style="display:none;">
      <h3>WEB сервисы</h3>
      <div class="admin-intro">
-        <b>Как читать статусы:</b> в строке сервиса <b>прогрето X / Y</b> и <b>занято: N</b> относятся к этому конкретному сервису.
+        <b>Как читать статусы:</b> прогрето X / Y — это состояние общего WEB-пула, занято: N — активные сессии конкретного сервиса.
+      </div>
+      <div class="panel" style="padding:0.8rem;">
+        <div class="list-title">Настройки общего WEB pool</div>
+        <div class="actions">
+          <input id="web_pool_size" type="number" min="0" value="{{ web_pool_size }}" style="max-width:220px;" />
+          <button onclick="saveWebPoolSize()">Save WEB pool size</button>
+        </div>
+        <small>Автодоращивание: active + {{ web_pool_buffer }}</small>
      </div>
      <div class="summary-strip">
        <div class="summary-card">
@@ -112,7 +120,7 @@
          <input class="list-search" id="web_search" placeholder="Поиск WEB сервиса..." oninput="filterList('web_search', '#web_list .web-item')" />
          <div class="list-box" id="web_list">
            {% for s in web_services %}
-              <button class="list-item service-row web-item" data-service-id="{{s.id}}" data-filter="{{(s.name ~ ' ' ~ s.slug)|lower}}" onclick='selectWebService({{s.id}}, {{s.name|tojson}}, {{s.slug|tojson}}, {{s.target|tojson}}, {{s.comment|tojson}}, {{s.icon_path|tojson}}, {{s.active|tojson}}, {{s.warm_pool_size}})'>
+              <button class="list-item service-row web-item" data-service-id="{{s.id}}" data-filter="{{(s.name ~ ' ' ~ s.slug)|lower}}" onclick='selectWebService({{s.id}}, {{s.name|tojson}}, {{s.slug|tojson}}, {{s.target|tojson}}, {{s.comment|tojson}}, {{s.icon_path|tojson}}, {{s.active|tojson}})'>
                <img class="service-thumb" src="{{ s.icon_path or '/static/service-placeholder.svg' }}" alt="icon" />
                <div>
                  <div>{{s.name}}</div>
@@ -147,11 +155,6 @@
              <textarea id="w_comment" placeholder="Коротко: что это за сервис"></textarea>
            </label>

-            <label class="field-col">
-              <span>Сколько держать прогретых</span>
-              <input id="w_pool" type="number" min="0" placeholder="Например: 2" />
-            </label>
-
            <label class="field-col">
              <span>Статус</span>
              <select id="w_active"><option value="true">active</option><option value="false">inactive</option></select>
@@ -159,7 +162,6 @@
          </div>
          <div class="actions">
            <button onclick="saveWebService()">Save</button>
-            <button onclick="prewarmNow('w_id')">Prewarm now</button>
            <button onclick="deleteService('w_id')">Delete</button>
            <button onclick="clearWebForm()">Clear</button>
          </div>
@@ -204,7 +206,7 @@

          <hr>
          <div class="list-title">Добавить WEB</div>
-          <div class="field-help">Рекомендуется начинать с pool size = 2-3 для часто используемых сервисов.</div>
+          <div class="field-help">Горячий пул задается сверху и общий для всех WEB сервисов.</div>
          <div class="form-grid labelled-grid">
            <input id="new_w_slug" type="hidden" />

@@ -223,11 +225,6 @@
              <textarea id="new_w_comment" placeholder="Коротко: что это за сервис"></textarea>
            </label>

-            <label class="field-col">
-              <span>Сколько держать прогретых</span>
-              <input id="new_w_pool" type="number" min="0" value="2" placeholder="Например: 2" />
-            </label>
-
            <label class="field-col">
              <span>Статус</span>
              <select id="new_w_active"><option value="true">active</option><option value="false">inactive</option></select>
@@ -593,20 +590,25 @@
      location.reload();
    }

-    function selectWebService(id, name, slug, target, comment, iconPath, active, pool) {
+    function selectWebService(id, name, slug, target, comment, iconPath, active) {
      document.getElementById('w_id').value = id;
      document.getElementById('w_name').value = name;
      document.getElementById('w_slug').value = slug;
      document.getElementById('w_target').value = target;
      document.getElementById('w_comment').value = comment || '';
      document.getElementById('w_active').value = String(active);
-      document.getElementById('w_pool').value = pool;
      document.getElementById('w_icon_preview').src = iconPath || placeholderIcon;
      document.getElementById('w_health_box').style.display = 'block';
      markSelected('.web-item', 'data-service-id', id);
      refreshSelectedServiceStatus('web');
    }

+    async function saveWebPoolSize() {
+      const size = parseInt(document.getElementById('web_pool_size').value || '0', 10);
+      await api('/api/admin/web-pool-size', 'PUT', {size});
+      location.reload();
+    }
+
    async function createWebService() {
      const slug = document.getElementById('new_w_slug').value || slugifyRu(document.getElementById('new_w_name').value);
      await api('/api/admin/services', 'POST', {
@@ -615,7 +617,6 @@
        type: 'WEB',
        target: document.getElementById('new_w_target').value,
        comment: document.getElementById('new_w_comment').value,
-        warm_pool_size: parseInt(document.getElementById('new_w_pool').value || '0', 10),
        active: document.getElementById('new_w_active').value === 'true',
      });
      location.reload();
@@ -631,14 +632,13 @@
        type: 'WEB',
        target: document.getElementById('w_target').value,
        comment: document.getElementById('w_comment').value,
-        warm_pool_size: parseInt(document.getElementById('w_pool').value || '0', 10),
        active: document.getElementById('w_active').value === 'true',
      });
      location.reload();
    }

    function clearWebForm() {
-      ['w_id','w_name','w_slug','w_target','w_comment','w_pool'].forEach(id => document.getElementById(id).value = '');
+      ['w_id','w_name','w_slug','w_target','w_comment'].forEach(id => document.getElementById(id).value = '');
      document.getElementById('w_active').value = 'true';
      document.getElementById('w_icon_preview').src = placeholderIcon;
      document.getElementById('w_health_box').style.display = 'none';