feat: improve session limit handling and add k6 load testing

app/main.py

@@ -1,6 +1,7 @@
 import datetime as dt
 import enum
 import fcntl
+import json
 import re
 import logging
 import os
@@ -9,13 +10,14 @@ import secrets
 import threading
 import time
 import uuid
+import contextvars
 from urllib.parse import parse_qs, unquote, urlparse
 from typing import Optional
 
 import docker
 import requests
 from fastapi import Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from itsdangerous import BadSignature, URLSafeTimedSerializer
@@ -44,6 +46,7 @@ COOKIE_MAX_AGE = 8 * 60 * 60
 SESSION_IDLE_SECONDS = int(os.getenv("SESSION_IDLE_SECONDS", "300"))
 PUBLIC_HOST = os.getenv("PUBLIC_HOST", "stend.4mont.ru")
 LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
+LOG_SLOW_REQUEST_MS = int(os.getenv("LOG_SLOW_REQUEST_MS", "2000"))
 TRAEFIK_INTERNAL_URL = os.getenv("TRAEFIK_INTERNAL_URL", "http://traefik")
 PREWARM_POOL_SIZE = int(os.getenv("PREWARM_POOL_SIZE", "0"))
 UNIVERSAL_POOL_SIZE = int(os.getenv("UNIVERSAL_POOL_SIZE", "0"))
@@ -63,6 +66,22 @@ logging.basicConfig(
     format="%(asctime)s %(levelname)s %(name)s %(message)s",
 )
 logger = logging.getLogger("portal")
+request_id_ctx = contextvars.ContextVar("request_id", default="-")
+
+
+def _normalize_log_value(value):
+    if isinstance(value, (str, int, float, bool)) or value is None:
+        return value
+    if isinstance(value, dt.datetime):
+        return value.isoformat()
+    return str(value)
+
+
+def log_event(event: str, level: int = logging.INFO, **fields) -> None:
+    payload = {"event": event, "req_id": request_id_ctx.get()}
+    for key, value in fields.items():
+        payload[key] = _normalize_log_value(value)
+    logger.log(level, json.dumps(payload, ensure_ascii=False, separators=(",", ":")))
 
 SIGNING_KEY = os.getenv("SIGNING_KEY", secrets.token_urlsafe(32))
 serializer = URLSafeTimedSerializer(SIGNING_KEY, salt="portal-auth")
@@ -79,22 +98,51 @@ app.mount("/static", StaticFiles(directory="static"), name="static")
 @app.middleware("http")
 async def request_logging_middleware(request: Request, call_next):
     req_id = request.headers.get("X-Request-ID", str(uuid.uuid4())[:8])
+    token = request_id_ctx.set(req_id)
     started = time.time()
+    client_ip = request.client.host if request.client else "-"
+    user_agent = request.headers.get("user-agent", "-")
     try:
         response = await call_next(request)
     except Exception:
-        logger.exception("request_failed req_id=%s method=%s path=%s", req_id, request.method, request.url.path)
+        log_event(
+            "request_failed",
+            level=logging.ERROR,
+            method=request.method,
+            path=request.url.path,
+            client_ip=client_ip,
+            user_agent=user_agent,
+        )
+        request_id_ctx.reset(token)
         raise
     duration_ms = int((time.time() - started) * 1000)
-    logger.info(
+    level = logging.INFO
-        "request req_id=%s method=%s path=%s status=%s duration_ms=%s",
+    if response.status_code >= 500:
-        req_id,
+        level = logging.ERROR
-        request.method,
+    elif response.status_code >= 400:
-        request.url.path,
+        level = logging.WARNING
-        response.status_code,
+    log_event(
-        duration_ms,
+        "request",
+        level=level,
+        method=request.method,
+        path=request.url.path,
+        query=str(request.url.query or ""),
+        status=response.status_code,
+        duration_ms=duration_ms,
+        client_ip=client_ip,
+        user_agent=user_agent,
+    )
+    if duration_ms >= LOG_SLOW_REQUEST_MS:
+        log_event(
+            "slow_request",
+            level=logging.WARNING,
+            method=request.method,
+            path=request.url.path,
+            duration_ms=duration_ms,
+            threshold_ms=LOG_SLOW_REQUEST_MS,
         )
     response.headers["X-Request-ID"] = req_id
+    request_id_ctx.reset(token)
     return response
 
 
@@ -112,6 +160,7 @@ class SessionStatus(str, enum.Enum):
     ACTIVE = "ACTIVE"
     EXPIRED = "EXPIRED"
     TERMINATED = "TERMINATED"
+    ROTATED = "ROTATED"
 
 
 class User(Base):
@@ -196,6 +245,28 @@ def now_utc() -> dt.datetime:
     return dt.datetime.now(dt.timezone.utc)
 
 
+def session_closed_reason(sess: SessionModel, db: Session) -> str:
+    if not sess:
+        return "idle"
+    if sess.status == SessionStatus.EXPIRED:
+        return "idle"
+    if sess.status == SessionStatus.ROTATED:
+        return "limit"
+    if sess.status == SessionStatus.TERMINATED:
+        cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS)
+        active_rows = db.scalars(
+            select(SessionModel).where(
+                SessionModel.user_id == sess.user_id,
+                SessionModel.status == SessionStatus.ACTIVE,
+                SessionModel.last_access_at >= cutoff,
+            )
+        ).all()
+        active_service_ids = {row.service_id for row in active_rows}
+        if len(active_service_ids) >= MAX_ACTIVE_SERVICES_PER_USER and sess.service_id not in active_service_ids:
+            return "limit"
+    return "idle"
+
+
 def normalize_web_target(url: str) -> str:
     raw = (url or "").strip()
     if not raw:
@@ -981,11 +1052,24 @@ def terminate_session_record(
 ) -> None:
     if not sess or sess.status != SessionStatus.ACTIVE:
         return
+    old_status = sess.status
     cid = sess.container_id or ""
     if stop_container and cid and not cid.startswith(("POOL:", "POOLIDX:", "WEBPOOLIDX:")):
         stop_runtime_container(cid)
     sess.status = new_status
     sess.last_access_at = now_utc()
+    log_event(
+        "session_closed",
+        level=logging.INFO,
+        session_id=sess.id,
+        user_id=sess.user_id,
+        service_id=sess.service_id,
+        container_id=cid,
+        old_status=old_status.value if isinstance(old_status, SessionStatus) else str(old_status),
+        new_status=new_status.value,
+        reason=session_closed_reason(sess, db),
+        stop_container=stop_container,
+    )
 
 
 def ensure_schema_compatibility() -> None:
@@ -1005,6 +1089,20 @@ def ensure_schema_compatibility() -> None:
                 """
             )
         )
+        conn.execute(
+            text(
+                """
+                DO $$
+                BEGIN
+                    BEGIN
+                        ALTER TYPE sessionstatus ADD VALUE IF NOT EXISTS 'ROTATED';
+                    EXCEPTION WHEN undefined_object THEN
+                        NULL;
+                    END;
+                END $$;
+                """
+            )
+        )
 
     with engine.begin() as conn:
         conn.execute(text("ALTER TABLE services ADD COLUMN IF NOT EXISTS warm_pool_size INTEGER NOT NULL DEFAULT 0"))
@@ -1430,6 +1528,11 @@ def index(request: Request, user: Optional[User] = Depends(get_current_user), db
     session_notice = ""
     if session_closed == "idle":
         session_notice = "Сессия была закрыта из-за простоя. Откройте сервис заново."
+    elif session_closed == "limit":
+        session_notice = (
+            f"Сессия была закрыта из-за лимита в {MAX_ACTIVE_SERVICES_PER_USER} сервиса(ов). "
+            "Освободите один сервис и попробуйте снова."
+        )
     elif launch_error == "max_services":
         session_notice = (
             f"Есть ограничение на {MAX_ACTIVE_SERVICES_PER_USER} сервиса(ов). "
@@ -1675,6 +1778,7 @@ def logout(request: Request):
 
 @app.get("/go/{slug}")
 def go_service(slug: str, user: User = Depends(require_user), db: Session = Depends(get_db)):
+    log_event("session_open_requested", user_id=user.id, service_slug=slug)
     service = db.scalar(select(Service).where(Service.slug == slug, Service.active == True))
     if not service:
         raise HTTPException(status_code=404, detail="Service not found")
@@ -1682,7 +1786,7 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
         raise HTTPException(status_code=410, detail="VNC services are deprecated")
     if not has_access(db, user.id, service.id):
         raise HTTPException(status_code=403, detail="ACL denied")
-
+    with allocator_lock(db, 92000 + int(user.id)):
         existing_user_session = find_active_session_for_user_service(db, user.id, service.id)
         if existing_user_session:
             return RedirectResponse(url=session_redirect_url(existing_user_session), status_code=303)
@@ -1700,14 +1804,14 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
         if service.id not in active_service_ids and len(active_service_ids) >= MAX_ACTIVE_SERVICES_PER_USER:
             oldest = next((row for row in active_rows if row.service_id != service.id), None)
             if oldest:
-            terminate_session_record(db, oldest, SessionStatus.TERMINATED, stop_container=True)
+                terminate_session_record(db, oldest, SessionStatus.ROTATED, stop_container=True)
                 db.commit()
-            logger.info(
+                log_event(
-                "session_rotated user_id=%s closed_session=%s old_service_id=%s new_service_id=%s",
+                    "session_rotated",
-                user.id,
+                    user_id=user.id,
-                oldest.id,
+                    closed_session_id=oldest.id,
-                oldest.service_id,
+                    closed_service_id=oldest.service_id,
-                service.id,
+                    new_service_id=service.id,
                 )
             else:
                 return RedirectResponse(url="/?launch_error=max_services", status_code=303)
@@ -1746,8 +1850,10 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
                     db.commit()
             except Exception as exc:
                 logger.exception("web_pool_dispatch_failed slug=%s user_id=%s", slug, user.id)
+                log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="web_pool", error=str(exc))
                 audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
                 raise HTTPException(status_code=502, detail="WEB runtime failed to switch target")
+            log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="web_pool", slot=slot)
             audit(db, "SESSION_CREATE_WEB_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id)
             return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
 
@@ -1772,8 +1878,10 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
                     db.commit()
             except Exception as exc:
                 logger.exception("universal_pool_dispatch_failed slug=%s user_id=%s", slug, user.id)
+                log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="universal_pool", error=str(exc))
                 audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
                 raise HTTPException(status_code=502, detail="Universal runtime failed to switch target")
+            log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="universal_pool", slot=slot)
             audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id)
             return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
 
@@ -1791,6 +1899,7 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
             )
             db.add(session_obj)
             db.commit()
+            log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="warm_pool")
             audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id}", user_id=user.id)
             return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
 
@@ -1798,6 +1907,7 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
             container_id = create_runtime_container(service, session_id)
         except Exception as exc:
             logger.exception("session_container_create_failed slug=%s user_id=%s", slug, user.id)
+            log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="single_runtime", error=str(exc))
             audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id)
             raise HTTPException(status_code=502, detail="Session runtime failed to start")
 
@@ -1812,10 +1922,11 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe
         )
         db.add(session_obj)
         db.commit()
+        log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="single_runtime", container_id=container_id)
 
         audit(db, "SESSION_CREATE", f"service={service.slug} session={session_id}", user_id=user.id)
         ready = wait_for_session_route(session_id)
-    logger.info("session_route_ready session_id=%s ready=%s", session_id, ready)
+        log_event("session_route_ready", session_id=session_id, ready=ready)
         return RedirectResponse(url=f"/s/{session_id}/", status_code=303)
 
 
@@ -1985,7 +2096,23 @@ def touch_session(session_id: str, user: User = Depends(require_user), db: Sessi
     if not sess or sess.user_id != user.id:
         raise HTTPException(status_code=404, detail="Session not found")
     if sess.status != SessionStatus.ACTIVE:
-        raise HTTPException(status_code=410, detail="Session expired")
+        reason = session_closed_reason(sess, db)
+        log_event(
+            "session_touch_rejected",
+            level=logging.WARNING,
+            session_id=session_id,
+            user_id=user.id,
+            status=sess.status.value,
+            reason=reason,
+        )
+        return JSONResponse(
+            status_code=410,
+            content={
+                "ok": False,
+                "reason": reason,
+                "status": sess.status.value,
+            },
+        )
     sess.last_access_at = now_utc()
     db.commit()
     return {"ok": True}
@@ -1997,9 +2124,17 @@ def close_session(session_id: str, user: User = Depends(require_user), db: Sessi
     if not sess or sess.user_id != user.id:
         raise HTTPException(status_code=404, detail="Session not found")
     if sess.status != SessionStatus.ACTIVE:
+        log_event(
+            "session_close_already_closed",
+            session_id=session_id,
+            user_id=user.id,
+            status=sess.status.value,
+            reason=session_closed_reason(sess, db),
+        )
         return {"ok": True, "status": sess.status.value}
     terminate_session_record(db, sess, SessionStatus.TERMINATED, stop_container=True)
     db.commit()
+    log_event("session_closed_by_user", session_id=session_id, user_id=user.id)
     return {"ok": True, "status": "TERMINATED"}
 
 

docker-compose.yml

@@ -30,7 +30,7 @@ services:
   api:
     build:
       context: ./app
-    command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "6"]
+    command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "18"]
     environment:
       DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
       SIGNING_KEY: ${SIGNING_KEY}

docs/LOAD_TESTING.md

@@ -0,0 +1,70 @@
+# Load Testing (k6)
+
+Готовый скрипт: `scripts/load/portal_k6.js`.
+
+## 1) Установка k6
+
+Ubuntu/Debian:
+
+```bash
+sudo gpg -k
+sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
+echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
+sudo apt update && sudo apt install -y k6
+```
+
+## 2) Smoke тест (быстрая проверка)
+
+```bash
+k6 run scripts/load/portal_k6.js \
+  -e BASE_URL=https://stend.4mont.ru \
+  -e USERNAME=ruslan \
+  -e PASSWORD='YOUR_PASSWORD' \
+  -e SERVICE_SLUGS=vmmanager,termidesk \
+  -e PROFILE=smoke
+```
+
+## 3) Нормальная нагрузка
+
+```bash
+k6 run scripts/load/portal_k6.js \
+  -e BASE_URL=https://stend.4mont.ru \
+  -e USERS_CSV='user1:pass1;user2:pass2;user3:pass3' \
+  -e SERVICE_SLUGS=vmmanager,termidesk,sayt-mont5 \
+  -e PROFILE=load \
+  -e HEARTBEATS=3 \
+  -e THINK_TIME=1
+```
+
+## 4) Стресс
+
+```bash
+k6 run scripts/load/portal_k6.js \
+  -e BASE_URL=https://stend.4mont.ru \
+  -e USERS_CSV='user1:pass1;user2:pass2;user3:pass3;user4:pass4' \
+  -e SERVICE_SLUGS=vmmanager,termidesk,sayt-mont5 \
+  -e PROFILE=stress
+```
+
+## 5) Ключевые параметры
+
+- `BASE_URL` — адрес портала.
+- `USERNAME`/`PASSWORD` — одиночный пользователь.
+- `USERS_CSV` — набор пользователей `u1:p1;u2:p2`.
+- `SERVICE_SLUGS` — список slug через запятую.
+- `PROFILE` — `smoke|load|stress`.
+- `HEARTBEATS` — сколько `touch` в итерации.
+- `THINK_TIME` — пауза между итерациями.
+- `CLOSE_SESSION=0` — не закрывать сессию в конце итерации.
+
+## 6) Что смотреть в результате
+
+- `http_req_duration` (`p95/p99`)
+- `http_req_failed`
+- `flow_errors`
+- `open_success`, `open_rejected`, `limit_redirects`, `touch_rejected`
+
+## 7) Важное
+
+- Запускайте сначала `smoke`, потом `load`, и только затем `stress`.
+- Для реалистичного теста используйте несколько пользователей, иначе упретесь в лимиты по сессиям одного пользователя.

kiosk/entrypoint.sh

@@ -59,22 +59,31 @@ cat > /opt/portal/index.html <<HTML
     rfb.scaleViewport = true;
     rfb.resizeSession = true;
     const enableHeartbeat = "${ENABLE_HEARTBEAT}" === "1";
-    const SESSION_CLOSED_URL = '/?session_closed=idle';
+    const SESSION_CLOSED_URL_BASE = '/?session_closed=';
     const CLOSE_PATH = '${TOUCH_PATH}'.replace(/\/touch$/, '/close');
-    function goSessionClosed() {
+    function goSessionClosed(reason = 'idle') {
+      const safeReason = reason === 'limit' ? 'limit' : 'idle';
+      const target = `${SESSION_CLOSED_URL_BASE}${safeReason}`;
       try {
         if (window.top && window.top !== window) {
-          window.top.location.href = SESSION_CLOSED_URL;
+          window.top.location.href = target;
           return;
         }
       } catch (e) {}
-      window.location.href = SESSION_CLOSED_URL;
+      window.location.href = target;
     }
     async function touch() {
       try {
         const res = await fetch('${TOUCH_PATH}', {method:'POST', credentials:'include'});
         if (!res.ok) {
-          goSessionClosed();
+          let reason = 'idle';
+          try {
+            const payload = await res.json();
+            if (payload && typeof payload.reason === 'string') {
+              reason = payload.reason;
+            }
+          } catch (e) {}
+          goSessionClosed(reason);
         }
       } catch(e) {}
     }

scripts/load/portal_k6.js

@@ -0,0 +1,220 @@
+import http from 'k6/http';
+import { check, fail, sleep } from 'k6';
+import exec from 'k6/execution';
+import { Counter, Rate, Trend } from 'k6/metrics';
+
+const BASE_URL = (__ENV.BASE_URL || 'https://stend.4mont.ru').replace(/\/$/, '');
+const SERVICE_SLUGS = (__ENV.SERVICE_SLUGS || 'vmmanager')
+  .split(',')
+  .map((s) => s.trim())
+  .filter(Boolean);
+const HEARTBEATS = Number(__ENV.HEARTBEATS || 3);
+const THINK_TIME = Number(__ENV.THINK_TIME || 1);
+const CLOSE_SESSION = (__ENV.CLOSE_SESSION || '1') !== '0';
+const PROFILE = (__ENV.PROFILE || 'smoke').toLowerCase();
+
+const usersFromCsv = parseUsersCsv(__ENV.USERS_CSV || '');
+const singleUser = {
+  username: __ENV.USERNAME || '',
+  password: __ENV.PASSWORD || '',
+};
+
+const openAttempts = new Counter('open_attempts');
+const openSuccess = new Counter('open_success');
+const openRejected = new Counter('open_rejected');
+const limitRedirects = new Counter('limit_redirects');
+const touchRejected = new Counter('touch_rejected');
+const closeCalls = new Counter('close_calls');
+const loginFailures = new Counter('login_failures');
+const flowErrors = new Rate('flow_errors');
+const openLatency = new Trend('open_latency_ms');
+
+const PROFILE_OPTIONS = {
+  smoke: {
+    vus: 5,
+    duration: '1m',
+  },
+  load: {
+    vus: 30,
+    duration: '10m',
+  },
+  stress: {
+    stages: [
+      { duration: '2m', target: 30 },
+      { duration: '5m', target: 80 },
+      { duration: '2m', target: 0 },
+    ],
+  },
+};
+
+const selected = PROFILE_OPTIONS[PROFILE] || PROFILE_OPTIONS.smoke;
+
+export const options = {
+  ...selected,
+  insecureSkipTLSVerify: (__ENV.INSECURE_TLS || '0') === '1',
+  thresholds: {
+    http_req_failed: ['rate<0.02'],
+    http_req_duration: ['p(95)<2000'],
+    flow_errors: ['rate<0.05'],
+  },
+};
+
+let loggedInKey = '';
+
+function parseUsersCsv(raw) {
+  if (!raw.trim()) return [];
+  return raw
+    .split(';')
+    .map((pair) => pair.trim())
+    .filter(Boolean)
+    .map((pair) => {
+      const idx = pair.indexOf(':');
+      if (idx <= 0) return null;
+      return {
+        username: pair.slice(0, idx).trim(),
+        password: pair.slice(idx + 1).trim(),
+      };
+    })
+    .filter((u) => u && u.username && u.password);
+}
+
+function currentCredentials() {
+  if (usersFromCsv.length > 0) {
+    const vu = exec.vu.idInTest || 1;
+    return usersFromCsv[(vu - 1) % usersFromCsv.length];
+  }
+  return singleUser;
+}
+
+function ensureLoggedIn() {
+  const creds = currentCredentials();
+  if (!creds.username || !creds.password) {
+    fail('Set USERNAME/PASSWORD or USERS_CSV (username:password;username2:password2)');
+  }
+
+  const userKey = `${creds.username}:${creds.password}`;
+  if (loggedInKey === userKey) return creds;
+
+  const jar = http.cookieJar();
+  jar.clear(BASE_URL);
+
+  const landing = http.get(`${BASE_URL}/`, { redirects: 0, tags: { name: 'GET /' } });
+  const csrfFromCookie = (jar.cookiesForURL(BASE_URL).csrf_token || [])[0] || '';
+  const csrf = csrfFromCookie || ((landing.cookies.csrf_token || [])[0] || {}).value || '';
+  if (!csrf) {
+    loginFailures.add(1);
+    fail('CSRF cookie not found on GET /');
+  }
+
+  const payload = [
+    `username=${encodeURIComponent(creds.username)}`,
+    `password=${encodeURIComponent(creds.password)}`,
+    `csrf_token=${encodeURIComponent(csrf)}`,
+  ].join('&');
+
+  const loginRes = http.post(`${BASE_URL}/login`, payload, {
+    redirects: 0,
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    tags: { name: 'POST /login' },
+  });
+
+  const ok = check(loginRes, {
+    'login status is redirect': (r) => r.status === 303,
+  });
+
+  const hasAuth = (jar.cookiesForURL(BASE_URL).portal_auth || []).length > 0;
+  if (!ok || !hasAuth) {
+    loginFailures.add(1);
+    fail(`Login failed for ${creds.username}: status=${loginRes.status}`);
+  }
+
+  loggedInKey = userKey;
+  return creds;
+}
+
+function pickSlug() {
+  return SERVICE_SLUGS[Math.floor(Math.random() * SERVICE_SLUGS.length)];
+}
+
+function extractSessionId(location) {
+  const match = (location || '').match(/\/s\/([0-9a-fA-F-]{36})\//);
+  return match ? match[1] : '';
+}
+
+export default function () {
+  ensureLoggedIn();
+
+  const slug = pickSlug();
+  openAttempts.add(1);
+
+  const openRes = http.get(`${BASE_URL}/go/${slug}`, {
+    redirects: 0,
+    tags: { name: 'GET /go/:slug' },
+  });
+  openLatency.add(openRes.timings.duration);
+
+  const location = openRes.headers.Location || '';
+  const sessionId = extractSessionId(location);
+
+  const opened = check(openRes, {
+    'open returns redirect': (r) => r.status === 303,
+  });
+
+  if (!opened) {
+    openRejected.add(1);
+    flowErrors.add(1);
+    sleep(THINK_TIME);
+    return;
+  }
+
+  if (location.includes('launch_error=max_services')) {
+    limitRedirects.add(1);
+    flowErrors.add(1);
+    sleep(THINK_TIME);
+    return;
+  }
+
+  if (!sessionId) {
+    openRejected.add(1);
+    flowErrors.add(1);
+    sleep(THINK_TIME);
+    return;
+  }
+
+  openSuccess.add(1);
+
+  for (let i = 0; i < HEARTBEATS; i += 1) {
+    const touchRes = http.post(`${BASE_URL}/api/sessions/${sessionId}/touch`, null, {
+      redirects: 0,
+      tags: { name: 'POST /api/sessions/:id/touch' },
+    });
+
+    if (touchRes.status === 410) {
+      touchRejected.add(1);
+      break;
+    }
+
+    const touchOk = check(touchRes, {
+      'touch status 200': (r) => r.status === 200,
+    });
+    if (!touchOk) {
+      flowErrors.add(1);
+      break;
+    }
+    sleep(0.5);
+  }
+
+  if (CLOSE_SESSION) {
+    const closeRes = http.post(`${BASE_URL}/api/sessions/${sessionId}/close`, null, {
+      redirects: 0,
+      tags: { name: 'POST /api/sessions/:id/close' },
+    });
+    closeCalls.add(1);
+    check(closeRes, {
+      'close status 200': (r) => r.status === 200,
+    });
+  }
+
+  flowErrors.add(0);
+  sleep(THINK_TIME);
+}

universal-runtime/entrypoint.sh

@@ -65,8 +65,15 @@ cat > /opt/portal/index.html <<'HTML'
     const statusEl = document.getElementById('status');
     const XK_ALT_L = 0xffe9;
     const XK_LEFT = 0xff51;
+
+    let rfb = null;
     let connected = false;
     let connectTimer = null;
+    let reconnectTimer = null;
+    let reconnectAttempts = 0;
+    const MAX_RECONNECT_ATTEMPTS = 12;
+    const RECONNECT_DELAYS_MS = [1000, 2000, 3000, 5000, 8000];
+    let manualDisconnect = false;
 
     function showStatus(text, isError = false) {
       statusEl.textContent = text;
@@ -78,73 +85,160 @@ cat > /opt/portal/index.html <<'HTML'
       statusEl.classList.add('hidden');
     }
 
-    showStatus('Подключение к слоту...');
+    function clearConnectTimer() {
+      if (connectTimer) {
+        clearTimeout(connectTimer);
+        connectTimer = null;
+      }
+    }
+
+    function clearReconnectTimer() {
+      if (reconnectTimer) {
+        clearTimeout(reconnectTimer);
+        reconnectTimer = null;
+      }
+    }
+
+    function reconnectDelay(attemptNumber) {
+      const idx = Math.min(Math.max(attemptNumber - 1, 0), RECONNECT_DELAYS_MS.length - 1);
+      return RECONNECT_DELAYS_MS[idx];
+    }
+
+    function scheduleConnectTimeout() {
+      clearConnectTimer();
       connectTimer = setTimeout(() => {
-      if (!connected) {
+        if (!connected && !manualDisconnect) {
-        showStatus('Нет подключения к экрану слота. Откройте сервис заново из дашборда.', true);
+          scheduleReconnect('Нет подключения к экрану слота. Пробуем переподключиться...');
         }
       }, 8000);
+    }
 
-    const rfb = new RFB(document.getElementById('screen'), wsUrl);
+    function scheduleReconnect(reasonText) {
+      if (manualDisconnect) return;
+      clearConnectTimer();
+      clearReconnectTimer();
+
+      if (reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {
+        showStatus('Соединение со слотом потеряно. Переподключение не удалось. Откройте сервис заново.', true);
+        return;
+      }
+
+      const nextAttempt = reconnectAttempts + 1;
+      const delayMs = reconnectDelay(nextAttempt);
+      const delaySec = Math.ceil(delayMs / 1000);
+      showStatus(`${reasonText} Повтор ${nextAttempt}/${MAX_RECONNECT_ATTEMPTS} через ${delaySec} сек.`, true);
+
+      reconnectTimer = setTimeout(() => {
+        reconnectAttempts = nextAttempt;
+        connectRfb('Переподключение к слоту...');
+      }, delayMs);
+    }
+
+    function attachRfb() {
+      if (rfb) {
+        try { rfb.disconnect(); } catch (e) {}
+      }
+      rfb = new RFB(document.getElementById('screen'), wsUrl);
       rfb.viewOnly = false;
       rfb.scaleViewport = true;
       rfb.resizeSession = true;
+
       rfb.addEventListener('connect', () => {
         connected = true;
-      if (connectTimer) clearTimeout(connectTimer);
+        reconnectAttempts = 0;
+        clearConnectTimer();
+        clearReconnectTimer();
         hideStatus();
       });
+
       rfb.addEventListener('disconnect', () => {
         connected = false;
-      showStatus('Соединение со слотом потеряно. Запустите сервис заново.', true);
+        if (manualDisconnect) return;
+        scheduleReconnect('Соединение со слотом потеряно.');
       });
+    }
+
+    function connectRfb(statusText) {
+      if (manualDisconnect) return;
+      connected = false;
+      showStatus(statusText || 'Подключение к слоту...');
+      attachRfb();
+      scheduleConnectTimeout();
+    }
 
     const enableHeartbeat = (new URLSearchParams(location.search).get('hb') ?? '1') !== '0';
     const sid = new URLSearchParams(location.search).get('sid');
-    const SESSION_CLOSED_URL = '/?session_closed=idle';
+    const SESSION_CLOSED_URL_BASE = '/?session_closed=';
     const CLOSE_PATH = sid ? `/api/sessions/${sid}/close` : '';
-    function goSessionClosed() {
+
+    function goSessionClosed(reason = 'idle') {
+      const safeReason = reason === 'limit' ? 'limit' : 'idle';
+      const target = `${SESSION_CLOSED_URL_BASE}${safeReason}`;
       try {
         if (window.top && window.top !== window) {
-          window.top.location.href = SESSION_CLOSED_URL;
+          window.top.location.href = target;
           return;
         }
       } catch (e) {}
-      window.location.href = SESSION_CLOSED_URL;
+      window.location.href = target;
     }
+
     async function touch() {
       if (!sid) return;
       try {
         const res = await fetch(`/api/sessions/${sid}/touch`, {method:'POST', credentials:'include'});
         if (!res.ok) {
-          goSessionClosed();
+          let reason = 'idle';
+          try {
+            const payload = await res.json();
+            if (payload && typeof payload.reason === 'string') {
+              reason = payload.reason;
+            }
+          } catch (e) {}
+          goSessionClosed(reason);
         }
       } catch (e) {}
     }
+
     let closing = false;
     async function closeSessionNow() {
       if (!CLOSE_PATH || closing) return;
       closing = true;
+      manualDisconnect = true;
+      clearConnectTimer();
+      clearReconnectTimer();
+      try {
+        if (rfb) rfb.disconnect();
+      } catch (e) {}
       try {
         await fetch(CLOSE_PATH, {method: 'POST', credentials: 'include', keepalive: true});
       } catch (e) {}
     }
+
     if (enableHeartbeat) {
       setInterval(touch, 15000);
       touch();
       window.addEventListener('pagehide', closeSessionNow);
       window.addEventListener('beforeunload', closeSessionNow);
     }
+
     function keyTap(keysym, code) {
+      if (!rfb) return;
       rfb.sendKey(keysym, code, true);
       rfb.sendKey(keysym, code, false);
     }
+
     function chord(mod, key, modCode, keyCode) {
+      if (!rfb) return;
       rfb.sendKey(mod, modCode, true);
       keyTap(key, keyCode);
       rfb.sendKey(mod, modCode, false);
     }
+
     function goHome() {
+      manualDisconnect = true;
+      clearConnectTimer();
+      clearReconnectTimer();
       try {
         if (window.top && window.top !== window) {
           window.top.location.href = '/';
@@ -153,9 +247,12 @@ cat > /opt/portal/index.html <<'HTML'
       } catch (e) {}
       window.location.href = '/';
     }
+
     document.getElementById('btn-back').addEventListener('click', () => chord(XK_ALT_L, XK_LEFT, 'AltLeft', 'ArrowLeft'));
     document.getElementById('btn-home').addEventListener('click', goHome);
     document.addEventListener('contextmenu', (e) => e.preventDefault());
+
+    connectRfb('Подключение к слоту...');
   </script>
 </body>
 </html>