feat: improve session limit handling and add k6 load testing

2026-04-23 05:17:53 +00:00
parent 47f46d5c5b
commit 1438dee21a
6 changed files with 687 additions and 156 deletions
@@ -1,6 +1,7 @@
 import datetime as dt
 import enum
 import fcntl
+import json
 import re
 import logging
 import os
@@ -9,13 +10,14 @@ import secrets
 import threading
 import time
 import uuid
+import contextvars
 from urllib.parse import parse_qs, unquote, urlparse
 from typing import Optional

 import docker
 import requests
 from fastapi import Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from itsdangerous import BadSignature, URLSafeTimedSerializer
@@ -44,6 +46,7 @@ COOKIE_MAX_AGE = 8 * 60 * 60
 SESSION_IDLE_SECONDS = int(os.getenv("SESSION_IDLE_SECONDS", "300"))
 PUBLIC_HOST = os.getenv("PUBLIC_HOST", "stend.4mont.ru")
 LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
+LOG_SLOW_REQUEST_MS = int(os.getenv("LOG_SLOW_REQUEST_MS", "2000"))
 TRAEFIK_INTERNAL_URL = os.getenv("TRAEFIK_INTERNAL_URL", "http://traefik")
 PREWARM_POOL_SIZE = int(os.getenv("PREWARM_POOL_SIZE", "0"))
 UNIVERSAL_POOL_SIZE = int(os.getenv("UNIVERSAL_POOL_SIZE", "0"))
@@ -63,6 +66,22 @@ logging.basicConfig(
    format="%(asctime)s %(levelname)s %(name)s %(message)s",
 )
 logger = logging.getLogger("portal")
+request_id_ctx = contextvars.ContextVar("request_id", default="-")
+
+
+def _normalize_log_value(value):
+    if isinstance(value, (str, int, float, bool)) or value is None:
+        return value
+    if isinstance(value, dt.datetime):
+        return value.isoformat()
+    return str(value)
+
+
+def log_event(event: str, level: int = logging.INFO, **fields) -> None:
+    payload = {"event": event, "req_id": request_id_ctx.get()}
+    for key, value in fields.items():
+        payload[key] = _normalize_log_value(value)
+    logger.log(level, json.dumps(payload, ensure_ascii=False, separators=(",", ":")))

 SIGNING_KEY = os.getenv("SIGNING_KEY", secrets.token_urlsafe(32))
 serializer = URLSafeTimedSerializer(SIGNING_KEY, salt="portal-auth")
@@ -79,22 +98,51 @@ app.mount("/static", StaticFiles(directory="static"), name="static")
@app.middleware("http")
 async def request_logging_middleware(request: Request, call_next):
    req_id = request.headers.get("X-Request-ID", str(uuid.uuid4())[:8])
+    token = request_id_ctx.set(req_id)
    started = time.time()
+    client_ip = request.client.host if request.client else "-"
+    user_agent = request.headers.get("user-agent", "-")
    try:
        response = await call_next(request)
    except Exception:
-        logger.exception("request_failed req_id=%s method=%s path=%s", req_id, request.method, request.url.path)
+        log_event(
+            "request_failed",
+            level=logging.ERROR,
+            method=request.method,
+            path=request.url.path,
+            client_ip=client_ip,
+            user_agent=user_agent,
+        )
+        request_id_ctx.reset(token)
        raise
    duration_ms = int((time.time() - started) * 1000)
-    logger.info(
-        "request req_id=%s method=%s path=%s status=%s duration_ms=%s",
-        req_id,
-        request.method,
-        request.url.path,
-        response.status_code,
-        duration_ms,
+    level = logging.INFO
+    if response.status_code >= 500:
+        level = logging.ERROR
+    elif response.status_code >= 400:
+        level = logging.WARNING
+    log_event(
+        "request",
+        level=level,
+        method=request.method,
+        path=request.url.path,
+        query=str(request.url.query or ""),
+        status=response.status_code,
+        duration_ms=duration_ms,
+        client_ip=client_ip,
+        user_agent=user_agent,
    )
+    if duration_ms >= LOG_SLOW_REQUEST_MS:
+        log_event(
+            "slow_request",
+            level=logging.WARNING,
+            method=request.method,
+            path=request.url.path,
+            duration_ms=duration_ms,
+            threshold_ms=LOG_SLOW_REQUEST_MS,
+        )
    response.headers["X-Request-ID"] = req_id
+    request_id_ctx.reset(token)
    return response


@@ -112,6 +160,7 @@ class SessionStatus(str, enum.Enum):
    ACTIVE = "ACTIVE"
    EXPIRED = "EXPIRED"
    TERMINATED = "TERMINATED"
+    ROTATED = "ROTATED"


 class User(Base):
@@ -196,6 +245,28 @@ def now_utc() -> dt.datetime:
    return dt.datetime.now(dt.timezone.utc)


+def session_closed_reason(sess: SessionModel, db: Session) -> str:
+    if not sess:
+        return "idle"
+    if sess.status == SessionStatus.EXPIRED:
+        return "idle"
+    if sess.status == SessionStatus.ROTATED:
+        return "limit"
+    if sess.status == SessionStatus.TERMINATED:
+        cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS)
+        active_rows = db.scalars(
+            select(SessionModel).where(
+                SessionModel.user_id == sess.user_id,
+                SessionModel.status == SessionStatus.ACTIVE,
+                SessionModel.last_access_at >= cutoff,
+            )
+        ).all()
+        active_service_ids = {row.service_id for row in active_rows}
+        if len(active_service_ids) >= MAX_ACTIVE_SERVICES_PER_USER and sess.service_id not in active_service_ids:
+            return "limit"
+    return "idle"
+
+
 def normalize_web_target(url: str) -> str:
    raw = (url or "").strip()
    if not raw:
@@ -981,11 +1052,24 @@ def terminate_session_record(
 ) -> None:
    if not sess or sess.status != SessionStatus.ACTIVE:
        return
+    old_status = sess.status
    cid = sess.container_id or ""
    if stop_container and cid and not cid.startswith(("POOL:", "POOLIDX:", "WEBPOOLIDX:")):
        stop_runtime_container(cid)
    sess.status = new_status
    sess.last_access_at = now_utc()
+    log_event(
+        "session_closed",
+        level=logging.INFO,
+        session_id=sess.id,
+        user_id=sess.user_id,
+        service_id=sess.service_id,
+        container_id=cid,
+        old_status=old_status.value if isinstance(old_status, SessionStatus) else str(old_status),
+        new_status=new_status.value,
+        reason=session_closed_reason(sess, db),
+        stop_container=stop_container,
+    )


 def ensure_schema_compatibility() -> None:
@@ -1005,6 +1089,20 @@ def ensure_schema_compatibility() -> None:
                """
            )
        )
+        conn.execute(
+            text(
+                """
+                DO $$
+                BEGIN
+                    BEGIN
+                        ALTER TYPE sessionstatus ADD VALUE IF NOT EXISTS 'ROTATED';
+                    EXCEPTION WHEN undefined_object THEN
+                        NULL;
+                    END;
+                END $$;
+                """
+            )
+        )

    with engine.begin() as conn:
        conn.execute(text("ALTER TABLE services ADD COLUMN IF NOT EXISTS warm_pool_size INTEGER NOT NULL DEFAULT 0"))
@@ -1430,6 +1528,11 @@ def index(request: Request, user: Optional[User] = Depends(get_current_user), db
    session_notice = ""
    if session_closed == "idle":
        session_notice = "Сессия была закрыта из-за простоя. Откройте сервис заново."
+    elif session_closed == "limit":
+        session_notice = (
+            f"Сессия была закрыта из-за лимита в {MAX_ACTIVE_SERVICES_PER_USER} сервиса(ов). "
+            "Освободите один сервис и попробуйте снова."
+        )
    elif launch_error == "max_services":
        session_notice = (
            f"Есть ограничение на {MAX_ACTIVE_SERVICES_PER_USER} сервиса(ов). "
@@ -1675,6 +1778,7 @@ def logout(request: Request):

@app.get("/go/{slug}")
 def go_service(slug: str, user: User = Depends(require_user), db: Session = Depends(get_db)):
+    log_event("session_open_requested", user_id=user.id, service_slug=slug)
    service = db.scalar(select(Service).where(Service.slug == slug, Service.active == True))
    if not service:
        raise HTTPException(status_code=404, detail="Service not found")
@@ -1682,142 +1786,149 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
        raise HTTPException(status_code=410, detail="VNC services are deprecated")
    if not has_access(db, user.id, service.id):
        raise HTTPException(status_code=403, detail="ACL denied")
+    with allocator_lock(db, 92000 + int(user.id)):
+        existing_user_session = find_active_session_for_user_service(db, user.id, service.id)
+        if existing_user_session:
+            return RedirectResponse(url=session_redirect_url(existing_user_session), status_code=303)

-    existing_user_session = find_active_session_for_user_service(db, user.id, service.id)
-    if existing_user_session:
-        return RedirectResponse(url=session_redirect_url(existing_user_session), status_code=303)
-
-    cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS)
-    active_rows = db.scalars(
-        select(SessionModel).where(
-            SessionModel.user_id == user.id,
-            SessionModel.status == SessionStatus.ACTIVE,
-            SessionModel.last_access_at >= cutoff,
-        )
-    ).all()
-    active_rows = sorted(active_rows, key=lambda row: row.created_at)
-    active_service_ids = {row.service_id for row in active_rows}
-    if service.id not in active_service_ids and len(active_service_ids) >= MAX_ACTIVE_SERVICES_PER_USER:
-        oldest = next((row for row in active_rows if row.service_id != service.id), None)
-        if oldest:
-            terminate_session_record(db, oldest, SessionStatus.TERMINATED, stop_container=True)
-            db.commit()
-            logger.info(
-                "session_rotated user_id=%s closed_session=%s old_service_id=%s new_service_id=%s",
-                user.id,
-                oldest.id,
-                oldest.service_id,
-                service.id,
+        cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS)
+        active_rows = db.scalars(
+            select(SessionModel).where(
+                SessionModel.user_id == user.id,
+                SessionModel.status == SessionStatus.ACTIVE,
+                SessionModel.last_access_at >= cutoff,
            )
-        else:
-            return RedirectResponse(url="/?launch_error=max_services", status_code=303)
-
-    if service.type == ServiceType.RDP:
-        active_owner = find_active_session_for_service(db, service.id)
-        if active_owner:
-            if active_owner.user_id != user.id:
-                owner = db.get(User, active_owner.user_id)
-                owner_name = owner.username if owner else f"id={active_owner.user_id}"
-                raise HTTPException(
-                    status_code=409,
-                    detail=f"RDP сервис уже занят пользователем {owner_name}. Попробуйте позже.",
-                )
-            return RedirectResponse(url=session_redirect_url(active_owner), status_code=303)
-
-    session_id = str(uuid.uuid4())
-    if service.type == ServiceType.WEB and WEB_POOL_SIZE > 0:
-        try:
-            with allocator_lock(db, 91001):
-                ensure_web_pool()
-                slot = acquire_web_pool_slot(db)
-                slot_cid = f"WEBPOOLIDX:{slot}"
-                terminate_active_slot_sessions(db, slot_cid)
-                dispatch_web_pool_target(slot, service)
-                session_obj = SessionModel(
-                    id=session_id,
-                    user_id=user.id,
-                    service_id=service.id,
-                    container_id=slot_cid,
-                    status=SessionStatus.ACTIVE,
-                    created_at=now_utc(),
-                    last_access_at=now_utc(),
-                )
-                db.add(session_obj)
+        ).all()
+        active_rows = sorted(active_rows, key=lambda row: row.created_at)
+        active_service_ids = {row.service_id for row in active_rows}
+        if service.id not in active_service_ids and len(active_service_ids) >= MAX_ACTIVE_SERVICES_PER_USER:
+            oldest = next((row for row in active_rows if row.service_id != service.id), None)
+            if oldest:
+                terminate_session_record(db, oldest, SessionStatus.ROTATED, stop_container=True)
                db.commit()
-        except Exception as exc:
-            logger.exception("web_pool_dispatch_failed slug=%s user_id=%s", slug, user.id)
-            audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
-            raise HTTPException(status_code=502, detail="WEB runtime failed to switch target")
-        audit(db, "SESSION_CREATE_WEB_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id)
-        return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
-
-    if service_uses_universal_pool(service):
-        try:
-            with allocator_lock(db, 91002):
-                ensure_universal_pool()
-                slot = acquire_universal_slot(db)
-                slot_cid = f"POOLIDX:{slot}"
-                terminate_active_slot_sessions(db, slot_cid)
-                dispatch_universal_target(slot, service)
-                session_obj = SessionModel(
-                    id=session_id,
+                log_event(
+                    "session_rotated",
                    user_id=user.id,
-                    service_id=service.id,
-                    container_id=slot_cid,
-                    status=SessionStatus.ACTIVE,
-                    created_at=now_utc(),
-                    last_access_at=now_utc(),
+                    closed_session_id=oldest.id,
+                    closed_service_id=oldest.service_id,
+                    new_service_id=service.id,
                )
-                db.add(session_obj)
-                db.commit()
-        except Exception as exc:
-            logger.exception("universal_pool_dispatch_failed slug=%s user_id=%s", slug, user.id)
-            audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
-            raise HTTPException(status_code=502, detail="Universal runtime failed to switch target")
-        audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id)
-        return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
+            else:
+                return RedirectResponse(url="/?launch_error=max_services", status_code=303)
+
+        if service.type == ServiceType.RDP:
+            active_owner = find_active_session_for_service(db, service.id)
+            if active_owner:
+                if active_owner.user_id != user.id:
+                    owner = db.get(User, active_owner.user_id)
+                    owner_name = owner.username if owner else f"id={active_owner.user_id}"
+                    raise HTTPException(
+                        status_code=409,
+                        detail=f"RDP сервис уже занят пользователем {owner_name}. Попробуйте позже.",
+                    )
+                return RedirectResponse(url=session_redirect_url(active_owner), status_code=303)
+
+        session_id = str(uuid.uuid4())
+        if service.type == ServiceType.WEB and WEB_POOL_SIZE > 0:
+            try:
+                with allocator_lock(db, 91001):
+                    ensure_web_pool()
+                    slot = acquire_web_pool_slot(db)
+                    slot_cid = f"WEBPOOLIDX:{slot}"
+                    terminate_active_slot_sessions(db, slot_cid)
+                    dispatch_web_pool_target(slot, service)
+                    session_obj = SessionModel(
+                        id=session_id,
+                        user_id=user.id,
+                        service_id=service.id,
+                        container_id=slot_cid,
+                        status=SessionStatus.ACTIVE,
+                        created_at=now_utc(),
+                        last_access_at=now_utc(),
+                    )
+                    db.add(session_obj)
+                    db.commit()
+            except Exception as exc:
+                logger.exception("web_pool_dispatch_failed slug=%s user_id=%s", slug, user.id)
+                log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="web_pool", error=str(exc))
+                audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
+                raise HTTPException(status_code=502, detail="WEB runtime failed to switch target")
+            log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="web_pool", slot=slot)
+            audit(db, "SESSION_CREATE_WEB_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id)
+            return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
+
+        if service_uses_universal_pool(service):
+            try:
+                with allocator_lock(db, 91002):
+                    ensure_universal_pool()
+                    slot = acquire_universal_slot(db)
+                    slot_cid = f"POOLIDX:{slot}"
+                    terminate_active_slot_sessions(db, slot_cid)
+                    dispatch_universal_target(slot, service)
+                    session_obj = SessionModel(
+                        id=session_id,
+                        user_id=user.id,
+                        service_id=service.id,
+                        container_id=slot_cid,
+                        status=SessionStatus.ACTIVE,
+                        created_at=now_utc(),
+                        last_access_at=now_utc(),
+                    )
+                    db.add(session_obj)
+                    db.commit()
+            except Exception as exc:
+                logger.exception("universal_pool_dispatch_failed slug=%s user_id=%s", slug, user.id)
+                log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="universal_pool", error=str(exc))
+                audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
+                raise HTTPException(status_code=502, detail="Universal runtime failed to switch target")
+            log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="universal_pool", slot=slot)
+            audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id)
+            return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
+
+        if service.type == ServiceType.WEB and desired_pool_size(service) > 0:
+            ensure_warm_pool(service)
+            open_warm_web_url(service, service.target)
+            session_obj = SessionModel(
+                id=session_id,
+                user_id=user.id,
+                service_id=service.id,
+                container_id=f"POOL:{service.slug}",
+                status=SessionStatus.ACTIVE,
+                created_at=now_utc(),
+                last_access_at=now_utc(),
+            )
+            db.add(session_obj)
+            db.commit()
+            log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="warm_pool")
+            audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id}", user_id=user.id)
+            return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
+
+        try:
+            container_id = create_runtime_container(service, session_id)
+        except Exception as exc:
+            logger.exception("session_container_create_failed slug=%s user_id=%s", slug, user.id)
+            log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="single_runtime", error=str(exc))
+            audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
+            raise HTTPException(status_code=502, detail="Session runtime failed to start")

-    if service.type == ServiceType.WEB and desired_pool_size(service) > 0:
-        ensure_warm_pool(service)
-        open_warm_web_url(service, service.target)
        session_obj = SessionModel(
            id=session_id,
            user_id=user.id,
            service_id=service.id,
-            container_id=f"POOL:{service.slug}",
+            container_id=container_id,
            status=SessionStatus.ACTIVE,
            created_at=now_utc(),
            last_access_at=now_utc(),
        )
        db.add(session_obj)
        db.commit()
-        audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id}", user_id=user.id)
+        log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="single_runtime", container_id=container_id)
+
+        audit(db, "SESSION_CREATE", f"service={service.slug} session={session_id}", user_id=user.id)
+        ready = wait_for_session_route(session_id)
+        log_event("session_route_ready", session_id=session_id, ready=ready)
        return RedirectResponse(url=f"/s/{session_id}/", status_code=303)

-    try:
-        container_id = create_runtime_container(service, session_id)
-    except Exception as exc:
-        logger.exception("session_container_create_failed slug=%s user_id=%s", slug, user.id)
-        audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
-        raise HTTPException(status_code=502, detail="Session runtime failed to start")
-
-    session_obj = SessionModel(
-        id=session_id,
-        user_id=user.id,
-        service_id=service.id,
-        container_id=container_id,
-        status=SessionStatus.ACTIVE,
-        created_at=now_utc(),
-        last_access_at=now_utc(),
-    )
-    db.add(session_obj)
-    db.commit()
-
-    audit(db, "SESSION_CREATE", f"service={service.slug} session={session_id}", user_id=user.id)
-    ready = wait_for_session_route(session_id)
-    logger.info("session_route_ready session_id=%s ready=%s", session_id, ready)
-    return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
-

@app.get("/svc/{slug}/", response_class=HTMLResponse)
 def service_wait_page(slug: str, request: Request, user: User = Depends(require_user), db: Session = Depends(get_db)):
@@ -1985,7 +2096,23 @@ def touch_session(session_id: str, user: User = Depends(require_user), db: Sessi
    if not sess or sess.user_id != user.id:
        raise HTTPException(status_code=404, detail="Session not found")
    if sess.status != SessionStatus.ACTIVE:
-        raise HTTPException(status_code=410, detail="Session expired")
+        reason = session_closed_reason(sess, db)
+        log_event(
+            "session_touch_rejected",
+            level=logging.WARNING,
+            session_id=session_id,
+            user_id=user.id,
+            status=sess.status.value,
+            reason=reason,
+        )
+        return JSONResponse(
+            status_code=410,
+            content={
+                "ok": False,
+                "reason": reason,
+                "status": sess.status.value,
+            },
+        )
    sess.last_access_at = now_utc()
    db.commit()
    return {"ok": True}
@@ -1997,9 +2124,17 @@ def close_session(session_id: str, user: User = Depends(require_user), db: Sessi
    if not sess or sess.user_id != user.id:
        raise HTTPException(status_code=404, detail="Session not found")
    if sess.status != SessionStatus.ACTIVE:
+        log_event(
+            "session_close_already_closed",
+            session_id=session_id,
+            user_id=user.id,
+            status=sess.status.value,
+            reason=session_closed_reason(sess, db),
+        )
        return {"ok": True, "status": sess.status.value}
    terminate_session_record(db, sess, SessionStatus.TERMINATED, stop_container=True)
    db.commit()
+    log_event("session_closed_by_user", session_id=session_id, user_id=user.id)
    return {"ok": True, "status": "TERMINATED"}


@@ -30,7 +30,7 @@ services:
  api:
    build:
      context: ./app
-    command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "6"]
+    command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "18"]
    environment:
      DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
      SIGNING_KEY: ${SIGNING_KEY}
@@ -0,0 +1,70 @@
+# Load Testing (k6)
+
+Готовый скрипт: `scripts/load/portal_k6.js`.
+
+## 1) Установка k6
+
+Ubuntu/Debian:
+
+```bash
+sudo gpg -k
+sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
+echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
+sudo apt update && sudo apt install -y k6
+```
+
+## 2) Smoke тест (быстрая проверка)
+
+```bash
+k6 run scripts/load/portal_k6.js \
+  -e BASE_URL=https://stend.4mont.ru \
+  -e USERNAME=ruslan \
+  -e PASSWORD='YOUR_PASSWORD' \
+  -e SERVICE_SLUGS=vmmanager,termidesk \
+  -e PROFILE=smoke
+```
+
+## 3) Нормальная нагрузка
+
+```bash
+k6 run scripts/load/portal_k6.js \
+  -e BASE_URL=https://stend.4mont.ru \
+  -e USERS_CSV='user1:pass1;user2:pass2;user3:pass3' \
+  -e SERVICE_SLUGS=vmmanager,termidesk,sayt-mont5 \
+  -e PROFILE=load \
+  -e HEARTBEATS=3 \
+  -e THINK_TIME=1
+```
+
+## 4) Стресс
+
+```bash
+k6 run scripts/load/portal_k6.js \
+  -e BASE_URL=https://stend.4mont.ru \
+  -e USERS_CSV='user1:pass1;user2:pass2;user3:pass3;user4:pass4' \
+  -e SERVICE_SLUGS=vmmanager,termidesk,sayt-mont5 \
+  -e PROFILE=stress
+```
+
+## 5) Ключевые параметры
+
+- `BASE_URL` — адрес портала.
+- `USERNAME`/`PASSWORD` — одиночный пользователь.
+- `USERS_CSV` — набор пользователей `u1:p1;u2:p2`.
+- `SERVICE_SLUGS` — список slug через запятую.
+- `PROFILE` — `smoke|load|stress`.
+- `HEARTBEATS` — сколько `touch` в итерации.
+- `THINK_TIME` — пауза между итерациями.
+- `CLOSE_SESSION=0` — не закрывать сессию в конце итерации.
+
+## 6) Что смотреть в результате
+
+- `http_req_duration` (`p95/p99`)
+- `http_req_failed`
+- `flow_errors`
+- `open_success`, `open_rejected`, `limit_redirects`, `touch_rejected`
+
+## 7) Важное
+
+- Запускайте сначала `smoke`, потом `load`, и только затем `stress`.
+- Для реалистичного теста используйте несколько пользователей, иначе упретесь в лимиты по сессиям одного пользователя.
@@ -59,22 +59,31 @@ cat > /opt/portal/index.html <<HTML
    rfb.scaleViewport = true;
    rfb.resizeSession = true;
    const enableHeartbeat = "${ENABLE_HEARTBEAT}" === "1";
-    const SESSION_CLOSED_URL = '/?session_closed=idle';
+    const SESSION_CLOSED_URL_BASE = '/?session_closed=';
    const CLOSE_PATH = '${TOUCH_PATH}'.replace(/\/touch$/, '/close');
-    function goSessionClosed() {
+    function goSessionClosed(reason = 'idle') {
+      const safeReason = reason === 'limit' ? 'limit' : 'idle';
+      const target = `${SESSION_CLOSED_URL_BASE}${safeReason}`;
      try {
        if (window.top && window.top !== window) {
-          window.top.location.href = SESSION_CLOSED_URL;
+          window.top.location.href = target;
          return;
        }
      } catch (e) {}
-      window.location.href = SESSION_CLOSED_URL;
+      window.location.href = target;
    }
    async function touch() {
      try {
        const res = await fetch('${TOUCH_PATH}', {method:'POST', credentials:'include'});
        if (!res.ok) {
-          goSessionClosed();
+          let reason = 'idle';
+          try {
+            const payload = await res.json();
+            if (payload && typeof payload.reason === 'string') {
+              reason = payload.reason;
+            }
+          } catch (e) {}
+          goSessionClosed(reason);
        }
      } catch(e) {}
    }
@@ -0,0 +1,220 @@
+import http from 'k6/http';
+import { check, fail, sleep } from 'k6';
+import exec from 'k6/execution';
+import { Counter, Rate, Trend } from 'k6/metrics';
+
+const BASE_URL = (__ENV.BASE_URL || 'https://stend.4mont.ru').replace(/\/$/, '');
+const SERVICE_SLUGS = (__ENV.SERVICE_SLUGS || 'vmmanager')
+  .split(',')
+  .map((s) => s.trim())
+  .filter(Boolean);
+const HEARTBEATS = Number(__ENV.HEARTBEATS || 3);
+const THINK_TIME = Number(__ENV.THINK_TIME || 1);
+const CLOSE_SESSION = (__ENV.CLOSE_SESSION || '1') !== '0';
+const PROFILE = (__ENV.PROFILE || 'smoke').toLowerCase();
+
+const usersFromCsv = parseUsersCsv(__ENV.USERS_CSV || '');
+const singleUser = {
+  username: __ENV.USERNAME || '',
+  password: __ENV.PASSWORD || '',
+};
+
+const openAttempts = new Counter('open_attempts');
+const openSuccess = new Counter('open_success');
+const openRejected = new Counter('open_rejected');
+const limitRedirects = new Counter('limit_redirects');
+const touchRejected = new Counter('touch_rejected');
+const closeCalls = new Counter('close_calls');
+const loginFailures = new Counter('login_failures');
+const flowErrors = new Rate('flow_errors');
+const openLatency = new Trend('open_latency_ms');
+
+const PROFILE_OPTIONS = {
+  smoke: {
+    vus: 5,
+    duration: '1m',
+  },
+  load: {
+    vus: 30,
+    duration: '10m',
+  },
+  stress: {
+    stages: [
+      { duration: '2m', target: 30 },
+      { duration: '5m', target: 80 },
+      { duration: '2m', target: 0 },
+    ],
+  },
+};
+
+const selected = PROFILE_OPTIONS[PROFILE] || PROFILE_OPTIONS.smoke;
+
+export const options = {
+  ...selected,
+  insecureSkipTLSVerify: (__ENV.INSECURE_TLS || '0') === '1',
+  thresholds: {
+    http_req_failed: ['rate<0.02'],
+    http_req_duration: ['p(95)<2000'],
+    flow_errors: ['rate<0.05'],
+  },
+};
+
+let loggedInKey = '';
+
+function parseUsersCsv(raw) {
+  if (!raw.trim()) return [];
+  return raw
+    .split(';')
+    .map((pair) => pair.trim())
+    .filter(Boolean)
+    .map((pair) => {
+      const idx = pair.indexOf(':');
+      if (idx <= 0) return null;
+      return {
+        username: pair.slice(0, idx).trim(),
+        password: pair.slice(idx + 1).trim(),
+      };
+    })
+    .filter((u) => u && u.username && u.password);
+}
+
+function currentCredentials() {
+  if (usersFromCsv.length > 0) {
+    const vu = exec.vu.idInTest || 1;
+    return usersFromCsv[(vu - 1) % usersFromCsv.length];
+  }
+  return singleUser;
+}
+
+function ensureLoggedIn() {
+  const creds = currentCredentials();
+  if (!creds.username || !creds.password) {
+    fail('Set USERNAME/PASSWORD or USERS_CSV (username:password;username2:password2)');
+  }
+
+  const userKey = `${creds.username}:${creds.password}`;
+  if (loggedInKey === userKey) return creds;
+
+  const jar = http.cookieJar();
+  jar.clear(BASE_URL);
+
+  const landing = http.get(`${BASE_URL}/`, { redirects: 0, tags: { name: 'GET /' } });
+  const csrfFromCookie = (jar.cookiesForURL(BASE_URL).csrf_token || [])[0] || '';
+  const csrf = csrfFromCookie || ((landing.cookies.csrf_token || [])[0] || {}).value || '';
+  if (!csrf) {
+    loginFailures.add(1);
+    fail('CSRF cookie not found on GET /');
+  }
+
+  const payload = [
+    `username=${encodeURIComponent(creds.username)}`,
+    `password=${encodeURIComponent(creds.password)}`,
+    `csrf_token=${encodeURIComponent(csrf)}`,
+  ].join('&');
+
+  const loginRes = http.post(`${BASE_URL}/login`, payload, {
+    redirects: 0,
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    tags: { name: 'POST /login' },
+  });
+
+  const ok = check(loginRes, {
+    'login status is redirect': (r) => r.status === 303,
+  });
+
+  const hasAuth = (jar.cookiesForURL(BASE_URL).portal_auth || []).length > 0;
+  if (!ok || !hasAuth) {
+    loginFailures.add(1);
+    fail(`Login failed for ${creds.username}: status=${loginRes.status}`);
+  }
+
+  loggedInKey = userKey;
+  return creds;
+}
+
+function pickSlug() {
+  return SERVICE_SLUGS[Math.floor(Math.random() * SERVICE_SLUGS.length)];
+}
+
+function extractSessionId(location) {
+  const match = (location || '').match(/\/s\/([0-9a-fA-F-]{36})\//);
+  return match ? match[1] : '';
+}
+
+export default function () {
+  ensureLoggedIn();
+
+  const slug = pickSlug();
+  openAttempts.add(1);
+
+  const openRes = http.get(`${BASE_URL}/go/${slug}`, {
+    redirects: 0,
+    tags: { name: 'GET /go/:slug' },
+  });
+  openLatency.add(openRes.timings.duration);
+
+  const location = openRes.headers.Location || '';
+  const sessionId = extractSessionId(location);
+
+  const opened = check(openRes, {
+    'open returns redirect': (r) => r.status === 303,
+  });
+
+  if (!opened) {
+    openRejected.add(1);
+    flowErrors.add(1);
+    sleep(THINK_TIME);
+    return;
+  }
+
+  if (location.includes('launch_error=max_services')) {
+    limitRedirects.add(1);
+    flowErrors.add(1);
+    sleep(THINK_TIME);
+    return;
+  }
+
+  if (!sessionId) {
+    openRejected.add(1);
+    flowErrors.add(1);
+    sleep(THINK_TIME);
+    return;
+  }
+
+  openSuccess.add(1);
+
+  for (let i = 0; i < HEARTBEATS; i += 1) {
+    const touchRes = http.post(`${BASE_URL}/api/sessions/${sessionId}/touch`, null, {
+      redirects: 0,
+      tags: { name: 'POST /api/sessions/:id/touch' },
+    });
+
+    if (touchRes.status === 410) {
+      touchRejected.add(1);
+      break;
+    }
+
+    const touchOk = check(touchRes, {
+      'touch status 200': (r) => r.status === 200,
+    });
+    if (!touchOk) {
+      flowErrors.add(1);
+      break;
+    }
+    sleep(0.5);
+  }
+
+  if (CLOSE_SESSION) {
+    const closeRes = http.post(`${BASE_URL}/api/sessions/${sessionId}/close`, null, {
+      redirects: 0,
+      tags: { name: 'POST /api/sessions/:id/close' },
+    });
+    closeCalls.add(1);
+    check(closeRes, {
+      'close status 200': (r) => r.status === 200,
+    });
+  }
+
+  flowErrors.add(0);
+  sleep(THINK_TIME);
+}
@@ -65,8 +65,15 @@ cat > /opt/portal/index.html <<'HTML'
    const statusEl = document.getElementById('status');
    const XK_ALT_L = 0xffe9;
    const XK_LEFT = 0xff51;
+
+    let rfb = null;
    let connected = false;
    let connectTimer = null;
+    let reconnectTimer = null;
+    let reconnectAttempts = 0;
+    const MAX_RECONNECT_ATTEMPTS = 12;
+    const RECONNECT_DELAYS_MS = [1000, 2000, 3000, 5000, 8000];
+    let manualDisconnect = false;

    function showStatus(text, isError = false) {
      statusEl.textContent = text;
@@ -78,73 +85,160 @@ cat > /opt/portal/index.html <<'HTML'
      statusEl.classList.add('hidden');
    }

-    showStatus('Подключение к слоту...');
-    connectTimer = setTimeout(() => {
-      if (!connected) {
-        showStatus('Нет подключения к экрану слота. Откройте сервис заново из дашборда.', true);
+    function clearConnectTimer() {
+      if (connectTimer) {
+        clearTimeout(connectTimer);
+        connectTimer = null;
      }
-    }, 8000);
+    }

-    const rfb = new RFB(document.getElementById('screen'), wsUrl);
-    rfb.viewOnly = false;
-    rfb.scaleViewport = true;
-    rfb.resizeSession = true;
-    rfb.addEventListener('connect', () => {
-      connected = true;
-      if (connectTimer) clearTimeout(connectTimer);
-      hideStatus();
-    });
-    rfb.addEventListener('disconnect', () => {
+    function clearReconnectTimer() {
+      if (reconnectTimer) {
+        clearTimeout(reconnectTimer);
+        reconnectTimer = null;
+      }
+    }
+
+    function reconnectDelay(attemptNumber) {
+      const idx = Math.min(Math.max(attemptNumber - 1, 0), RECONNECT_DELAYS_MS.length - 1);
+      return RECONNECT_DELAYS_MS[idx];
+    }
+
+    function scheduleConnectTimeout() {
+      clearConnectTimer();
+      connectTimer = setTimeout(() => {
+        if (!connected && !manualDisconnect) {
+          scheduleReconnect('Нет подключения к экрану слота. Пробуем переподключиться...');
+        }
+      }, 8000);
+    }
+
+    function scheduleReconnect(reasonText) {
+      if (manualDisconnect) return;
+      clearConnectTimer();
+      clearReconnectTimer();
+
+      if (reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {
+        showStatus('Соединение со слотом потеряно. Переподключение не удалось. Откройте сервис заново.', true);
+        return;
+      }
+
+      const nextAttempt = reconnectAttempts + 1;
+      const delayMs = reconnectDelay(nextAttempt);
+      const delaySec = Math.ceil(delayMs / 1000);
+      showStatus(`${reasonText} Повтор ${nextAttempt}/${MAX_RECONNECT_ATTEMPTS} через ${delaySec} сек.`, true);
+
+      reconnectTimer = setTimeout(() => {
+        reconnectAttempts = nextAttempt;
+        connectRfb('Переподключение к слоту...');
+      }, delayMs);
+    }
+
+    function attachRfb() {
+      if (rfb) {
+        try { rfb.disconnect(); } catch (e) {}
+      }
+      rfb = new RFB(document.getElementById('screen'), wsUrl);
+      rfb.viewOnly = false;
+      rfb.scaleViewport = true;
+      rfb.resizeSession = true;
+
+      rfb.addEventListener('connect', () => {
+        connected = true;
+        reconnectAttempts = 0;
+        clearConnectTimer();
+        clearReconnectTimer();
+        hideStatus();
+      });
+
+      rfb.addEventListener('disconnect', () => {
+        connected = false;
+        if (manualDisconnect) return;
+        scheduleReconnect('Соединение со слотом потеряно.');
+      });
+    }
+
+    function connectRfb(statusText) {
+      if (manualDisconnect) return;
      connected = false;
-      showStatus('Соединение со слотом потеряно. Запустите сервис заново.', true);
-    });
+      showStatus(statusText || 'Подключение к слоту...');
+      attachRfb();
+      scheduleConnectTimeout();
+    }

    const enableHeartbeat = (new URLSearchParams(location.search).get('hb') ?? '1') !== '0';
    const sid = new URLSearchParams(location.search).get('sid');
-    const SESSION_CLOSED_URL = '/?session_closed=idle';
+    const SESSION_CLOSED_URL_BASE = '/?session_closed=';
    const CLOSE_PATH = sid ? `/api/sessions/${sid}/close` : '';
-    function goSessionClosed() {
+
+    function goSessionClosed(reason = 'idle') {
+      const safeReason = reason === 'limit' ? 'limit' : 'idle';
+      const target = `${SESSION_CLOSED_URL_BASE}${safeReason}`;
      try {
        if (window.top && window.top !== window) {
-          window.top.location.href = SESSION_CLOSED_URL;
+          window.top.location.href = target;
          return;
        }
      } catch (e) {}
-      window.location.href = SESSION_CLOSED_URL;
+      window.location.href = target;
    }
+
    async function touch() {
      if (!sid) return;
      try {
        const res = await fetch(`/api/sessions/${sid}/touch`, {method:'POST', credentials:'include'});
        if (!res.ok) {
-          goSessionClosed();
+          let reason = 'idle';
+          try {
+            const payload = await res.json();
+            if (payload && typeof payload.reason === 'string') {
+              reason = payload.reason;
+            }
+          } catch (e) {}
+          goSessionClosed(reason);
        }
      } catch (e) {}
    }
+
    let closing = false;
    async function closeSessionNow() {
      if (!CLOSE_PATH || closing) return;
      closing = true;
+      manualDisconnect = true;
+      clearConnectTimer();
+      clearReconnectTimer();
+      try {
+        if (rfb) rfb.disconnect();
+      } catch (e) {}
      try {
        await fetch(CLOSE_PATH, {method: 'POST', credentials: 'include', keepalive: true});
      } catch (e) {}
    }
+
    if (enableHeartbeat) {
      setInterval(touch, 15000);
      touch();
      window.addEventListener('pagehide', closeSessionNow);
      window.addEventListener('beforeunload', closeSessionNow);
    }
+
    function keyTap(keysym, code) {
+      if (!rfb) return;
      rfb.sendKey(keysym, code, true);
      rfb.sendKey(keysym, code, false);
    }
+
    function chord(mod, key, modCode, keyCode) {
+      if (!rfb) return;
      rfb.sendKey(mod, modCode, true);
      keyTap(key, keyCode);
      rfb.sendKey(mod, modCode, false);
    }
+
    function goHome() {
+      manualDisconnect = true;
+      clearConnectTimer();
+      clearReconnectTimer();
      try {
        if (window.top && window.top !== window) {
          window.top.location.href = '/';
@@ -153,9 +247,12 @@ cat > /opt/portal/index.html <<'HTML'
      } catch (e) {}
      window.location.href = '/';
    }
+
    document.getElementById('btn-back').addEventListener('click', () => chord(XK_ALT_L, XK_LEFT, 'AltLeft', 'ArrowLeft'));
    document.getElementById('btn-home').addEventListener('click', goHome);
    document.addEventListener('contextmenu', (e) => e.preventDefault());
+
+    connectRfb('Подключение к слоту...');
  </script>
 </body>
 </html>