From 1438dee21ac7b58e15e2a32302dcb4b1e345b9ea Mon Sep 17 00:00:00 2001 From: Ruslan Date: Thu, 23 Apr 2026 05:17:53 +0000 Subject: [PATCH] feat: improve session limit handling and add k6 load testing --- app/main.py | 391 +++++++++++++++++++++----------- docker-compose.yml | 2 +- docs/LOAD_TESTING.md | 70 ++++++ kiosk/entrypoint.sh | 19 +- scripts/load/portal_k6.js | 220 ++++++++++++++++++ universal-runtime/entrypoint.sh | 141 ++++++++++-- 6 files changed, 687 insertions(+), 156 deletions(-) create mode 100644 docs/LOAD_TESTING.md create mode 100644 scripts/load/portal_k6.js diff --git a/app/main.py b/app/main.py index 563ba99..872d7e8 100644 --- a/app/main.py +++ b/app/main.py @@ -1,6 +1,7 @@ import datetime as dt import enum import fcntl +import json import re import logging import os @@ -9,13 +10,14 @@ import secrets import threading import time import uuid +import contextvars from urllib.parse import parse_qs, unquote, urlparse from typing import Optional import docker import requests from fastapi import Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status -from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from itsdangerous import BadSignature, URLSafeTimedSerializer @@ -44,6 +46,7 @@ COOKIE_MAX_AGE = 8 * 60 * 60 SESSION_IDLE_SECONDS = int(os.getenv("SESSION_IDLE_SECONDS", "300")) PUBLIC_HOST = os.getenv("PUBLIC_HOST", "stend.4mont.ru") LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper() +LOG_SLOW_REQUEST_MS = int(os.getenv("LOG_SLOW_REQUEST_MS", "2000")) TRAEFIK_INTERNAL_URL = os.getenv("TRAEFIK_INTERNAL_URL", "http://traefik") PREWARM_POOL_SIZE = int(os.getenv("PREWARM_POOL_SIZE", "0")) UNIVERSAL_POOL_SIZE = int(os.getenv("UNIVERSAL_POOL_SIZE", "0")) @@ -63,6 +66,22 @@ logging.basicConfig( format="%(asctime)s %(levelname)s %(name)s %(message)s", ) logger = logging.getLogger("portal") +request_id_ctx = contextvars.ContextVar("request_id", default="-") + + +def _normalize_log_value(value): + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, dt.datetime): + return value.isoformat() + return str(value) + + +def log_event(event: str, level: int = logging.INFO, **fields) -> None: + payload = {"event": event, "req_id": request_id_ctx.get()} + for key, value in fields.items(): + payload[key] = _normalize_log_value(value) + logger.log(level, json.dumps(payload, ensure_ascii=False, separators=(",", ":"))) SIGNING_KEY = os.getenv("SIGNING_KEY", secrets.token_urlsafe(32)) serializer = URLSafeTimedSerializer(SIGNING_KEY, salt="portal-auth") @@ -79,22 +98,51 @@ app.mount("/static", StaticFiles(directory="static"), name="static") @app.middleware("http") async def request_logging_middleware(request: Request, call_next): req_id = request.headers.get("X-Request-ID", str(uuid.uuid4())[:8]) + token = request_id_ctx.set(req_id) started = time.time() + client_ip = request.client.host if request.client else "-" + user_agent = request.headers.get("user-agent", "-") try: response = await call_next(request) except Exception: - logger.exception("request_failed req_id=%s method=%s path=%s", req_id, request.method, request.url.path) + log_event( + "request_failed", + level=logging.ERROR, + method=request.method, + path=request.url.path, + client_ip=client_ip, + user_agent=user_agent, + ) + request_id_ctx.reset(token) raise duration_ms = int((time.time() - started) * 1000) - logger.info( - "request req_id=%s method=%s path=%s status=%s duration_ms=%s", - req_id, - request.method, - request.url.path, - response.status_code, - duration_ms, + level = logging.INFO + if response.status_code >= 500: + level = logging.ERROR + elif response.status_code >= 400: + level = logging.WARNING + log_event( + "request", + level=level, + method=request.method, + path=request.url.path, + query=str(request.url.query or ""), + status=response.status_code, + duration_ms=duration_ms, + client_ip=client_ip, + user_agent=user_agent, ) + if duration_ms >= LOG_SLOW_REQUEST_MS: + log_event( + "slow_request", + level=logging.WARNING, + method=request.method, + path=request.url.path, + duration_ms=duration_ms, + threshold_ms=LOG_SLOW_REQUEST_MS, + ) response.headers["X-Request-ID"] = req_id + request_id_ctx.reset(token) return response @@ -112,6 +160,7 @@ class SessionStatus(str, enum.Enum): ACTIVE = "ACTIVE" EXPIRED = "EXPIRED" TERMINATED = "TERMINATED" + ROTATED = "ROTATED" class User(Base): @@ -196,6 +245,28 @@ def now_utc() -> dt.datetime: return dt.datetime.now(dt.timezone.utc) +def session_closed_reason(sess: SessionModel, db: Session) -> str: + if not sess: + return "idle" + if sess.status == SessionStatus.EXPIRED: + return "idle" + if sess.status == SessionStatus.ROTATED: + return "limit" + if sess.status == SessionStatus.TERMINATED: + cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS) + active_rows = db.scalars( + select(SessionModel).where( + SessionModel.user_id == sess.user_id, + SessionModel.status == SessionStatus.ACTIVE, + SessionModel.last_access_at >= cutoff, + ) + ).all() + active_service_ids = {row.service_id for row in active_rows} + if len(active_service_ids) >= MAX_ACTIVE_SERVICES_PER_USER and sess.service_id not in active_service_ids: + return "limit" + return "idle" + + def normalize_web_target(url: str) -> str: raw = (url or "").strip() if not raw: @@ -981,11 +1052,24 @@ def terminate_session_record( ) -> None: if not sess or sess.status != SessionStatus.ACTIVE: return + old_status = sess.status cid = sess.container_id or "" if stop_container and cid and not cid.startswith(("POOL:", "POOLIDX:", "WEBPOOLIDX:")): stop_runtime_container(cid) sess.status = new_status sess.last_access_at = now_utc() + log_event( + "session_closed", + level=logging.INFO, + session_id=sess.id, + user_id=sess.user_id, + service_id=sess.service_id, + container_id=cid, + old_status=old_status.value if isinstance(old_status, SessionStatus) else str(old_status), + new_status=new_status.value, + reason=session_closed_reason(sess, db), + stop_container=stop_container, + ) def ensure_schema_compatibility() -> None: @@ -1005,6 +1089,20 @@ def ensure_schema_compatibility() -> None: """ ) ) + conn.execute( + text( + """ + DO $$ + BEGIN + BEGIN + ALTER TYPE sessionstatus ADD VALUE IF NOT EXISTS 'ROTATED'; + EXCEPTION WHEN undefined_object THEN + NULL; + END; + END $$; + """ + ) + ) with engine.begin() as conn: conn.execute(text("ALTER TABLE services ADD COLUMN IF NOT EXISTS warm_pool_size INTEGER NOT NULL DEFAULT 0")) @@ -1430,6 +1528,11 @@ def index(request: Request, user: Optional[User] = Depends(get_current_user), db session_notice = "" if session_closed == "idle": session_notice = "Сессия была закрыта из-за простоя. Откройте сервис заново." + elif session_closed == "limit": + session_notice = ( + f"Сессия была закрыта из-за лимита в {MAX_ACTIVE_SERVICES_PER_USER} сервиса(ов). " + "Освободите один сервис и попробуйте снова." + ) elif launch_error == "max_services": session_notice = ( f"Есть ограничение на {MAX_ACTIVE_SERVICES_PER_USER} сервиса(ов). " @@ -1675,6 +1778,7 @@ def logout(request: Request): @app.get("/go/{slug}") def go_service(slug: str, user: User = Depends(require_user), db: Session = Depends(get_db)): + log_event("session_open_requested", user_id=user.id, service_slug=slug) service = db.scalar(select(Service).where(Service.slug == slug, Service.active == True)) if not service: raise HTTPException(status_code=404, detail="Service not found") @@ -1682,142 +1786,149 @@ def go_service(slug: str, user: User = Depends(require_user), db: Session = Depe raise HTTPException(status_code=410, detail="VNC services are deprecated") if not has_access(db, user.id, service.id): raise HTTPException(status_code=403, detail="ACL denied") + with allocator_lock(db, 92000 + int(user.id)): + existing_user_session = find_active_session_for_user_service(db, user.id, service.id) + if existing_user_session: + return RedirectResponse(url=session_redirect_url(existing_user_session), status_code=303) - existing_user_session = find_active_session_for_user_service(db, user.id, service.id) - if existing_user_session: - return RedirectResponse(url=session_redirect_url(existing_user_session), status_code=303) - - cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS) - active_rows = db.scalars( - select(SessionModel).where( - SessionModel.user_id == user.id, - SessionModel.status == SessionStatus.ACTIVE, - SessionModel.last_access_at >= cutoff, - ) - ).all() - active_rows = sorted(active_rows, key=lambda row: row.created_at) - active_service_ids = {row.service_id for row in active_rows} - if service.id not in active_service_ids and len(active_service_ids) >= MAX_ACTIVE_SERVICES_PER_USER: - oldest = next((row for row in active_rows if row.service_id != service.id), None) - if oldest: - terminate_session_record(db, oldest, SessionStatus.TERMINATED, stop_container=True) - db.commit() - logger.info( - "session_rotated user_id=%s closed_session=%s old_service_id=%s new_service_id=%s", - user.id, - oldest.id, - oldest.service_id, - service.id, + cutoff = now_utc() - dt.timedelta(seconds=SESSION_IDLE_SECONDS) + active_rows = db.scalars( + select(SessionModel).where( + SessionModel.user_id == user.id, + SessionModel.status == SessionStatus.ACTIVE, + SessionModel.last_access_at >= cutoff, ) - else: - return RedirectResponse(url="/?launch_error=max_services", status_code=303) - - if service.type == ServiceType.RDP: - active_owner = find_active_session_for_service(db, service.id) - if active_owner: - if active_owner.user_id != user.id: - owner = db.get(User, active_owner.user_id) - owner_name = owner.username if owner else f"id={active_owner.user_id}" - raise HTTPException( - status_code=409, - detail=f"RDP сервис уже занят пользователем {owner_name}. Попробуйте позже.", - ) - return RedirectResponse(url=session_redirect_url(active_owner), status_code=303) - - session_id = str(uuid.uuid4()) - if service.type == ServiceType.WEB and WEB_POOL_SIZE > 0: - try: - with allocator_lock(db, 91001): - ensure_web_pool() - slot = acquire_web_pool_slot(db) - slot_cid = f"WEBPOOLIDX:{slot}" - terminate_active_slot_sessions(db, slot_cid) - dispatch_web_pool_target(slot, service) - session_obj = SessionModel( - id=session_id, - user_id=user.id, - service_id=service.id, - container_id=slot_cid, - status=SessionStatus.ACTIVE, - created_at=now_utc(), - last_access_at=now_utc(), - ) - db.add(session_obj) + ).all() + active_rows = sorted(active_rows, key=lambda row: row.created_at) + active_service_ids = {row.service_id for row in active_rows} + if service.id not in active_service_ids and len(active_service_ids) >= MAX_ACTIVE_SERVICES_PER_USER: + oldest = next((row for row in active_rows if row.service_id != service.id), None) + if oldest: + terminate_session_record(db, oldest, SessionStatus.ROTATED, stop_container=True) db.commit() - except Exception as exc: - logger.exception("web_pool_dispatch_failed slug=%s user_id=%s", slug, user.id) - audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id) - raise HTTPException(status_code=502, detail="WEB runtime failed to switch target") - audit(db, "SESSION_CREATE_WEB_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id) - return RedirectResponse(url=f"/s/{session_id}/", status_code=303) - - if service_uses_universal_pool(service): - try: - with allocator_lock(db, 91002): - ensure_universal_pool() - slot = acquire_universal_slot(db) - slot_cid = f"POOLIDX:{slot}" - terminate_active_slot_sessions(db, slot_cid) - dispatch_universal_target(slot, service) - session_obj = SessionModel( - id=session_id, + log_event( + "session_rotated", user_id=user.id, - service_id=service.id, - container_id=slot_cid, - status=SessionStatus.ACTIVE, - created_at=now_utc(), - last_access_at=now_utc(), + closed_session_id=oldest.id, + closed_service_id=oldest.service_id, + new_service_id=service.id, ) - db.add(session_obj) - db.commit() - except Exception as exc: - logger.exception("universal_pool_dispatch_failed slug=%s user_id=%s", slug, user.id) - audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id) - raise HTTPException(status_code=502, detail="Universal runtime failed to switch target") - audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id) - return RedirectResponse(url=f"/s/{session_id}/", status_code=303) + else: + return RedirectResponse(url="/?launch_error=max_services", status_code=303) + + if service.type == ServiceType.RDP: + active_owner = find_active_session_for_service(db, service.id) + if active_owner: + if active_owner.user_id != user.id: + owner = db.get(User, active_owner.user_id) + owner_name = owner.username if owner else f"id={active_owner.user_id}" + raise HTTPException( + status_code=409, + detail=f"RDP сервис уже занят пользователем {owner_name}. Попробуйте позже.", + ) + return RedirectResponse(url=session_redirect_url(active_owner), status_code=303) + + session_id = str(uuid.uuid4()) + if service.type == ServiceType.WEB and WEB_POOL_SIZE > 0: + try: + with allocator_lock(db, 91001): + ensure_web_pool() + slot = acquire_web_pool_slot(db) + slot_cid = f"WEBPOOLIDX:{slot}" + terminate_active_slot_sessions(db, slot_cid) + dispatch_web_pool_target(slot, service) + session_obj = SessionModel( + id=session_id, + user_id=user.id, + service_id=service.id, + container_id=slot_cid, + status=SessionStatus.ACTIVE, + created_at=now_utc(), + last_access_at=now_utc(), + ) + db.add(session_obj) + db.commit() + except Exception as exc: + logger.exception("web_pool_dispatch_failed slug=%s user_id=%s", slug, user.id) + log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="web_pool", error=str(exc)) + audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id) + raise HTTPException(status_code=502, detail="WEB runtime failed to switch target") + log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="web_pool", slot=slot) + audit(db, "SESSION_CREATE_WEB_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id) + return RedirectResponse(url=f"/s/{session_id}/", status_code=303) + + if service_uses_universal_pool(service): + try: + with allocator_lock(db, 91002): + ensure_universal_pool() + slot = acquire_universal_slot(db) + slot_cid = f"POOLIDX:{slot}" + terminate_active_slot_sessions(db, slot_cid) + dispatch_universal_target(slot, service) + session_obj = SessionModel( + id=session_id, + user_id=user.id, + service_id=service.id, + container_id=slot_cid, + status=SessionStatus.ACTIVE, + created_at=now_utc(), + last_access_at=now_utc(), + ) + db.add(session_obj) + db.commit() + except Exception as exc: + logger.exception("universal_pool_dispatch_failed slug=%s user_id=%s", slug, user.id) + log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="universal_pool", error=str(exc)) + audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id) + raise HTTPException(status_code=502, detail="Universal runtime failed to switch target") + log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="universal_pool", slot=slot) + audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id} slot={slot}", user_id=user.id) + return RedirectResponse(url=f"/s/{session_id}/", status_code=303) + + if service.type == ServiceType.WEB and desired_pool_size(service) > 0: + ensure_warm_pool(service) + open_warm_web_url(service, service.target) + session_obj = SessionModel( + id=session_id, + user_id=user.id, + service_id=service.id, + container_id=f"POOL:{service.slug}", + status=SessionStatus.ACTIVE, + created_at=now_utc(), + last_access_at=now_utc(), + ) + db.add(session_obj) + db.commit() + log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="warm_pool") + audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id}", user_id=user.id) + return RedirectResponse(url=f"/s/{session_id}/", status_code=303) + + try: + container_id = create_runtime_container(service, session_id) + except Exception as exc: + logger.exception("session_container_create_failed slug=%s user_id=%s", slug, user.id) + log_event("session_create_failed", level=logging.ERROR, user_id=user.id, service_slug=slug, mode="single_runtime", error=str(exc)) + audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id) + raise HTTPException(status_code=502, detail="Session runtime failed to start") - if service.type == ServiceType.WEB and desired_pool_size(service) > 0: - ensure_warm_pool(service) - open_warm_web_url(service, service.target) session_obj = SessionModel( id=session_id, user_id=user.id, service_id=service.id, - container_id=f"POOL:{service.slug}", + container_id=container_id, status=SessionStatus.ACTIVE, created_at=now_utc(), last_access_at=now_utc(), ) db.add(session_obj) db.commit() - audit(db, "SESSION_CREATE_POOL", f"service={service.slug} session={session_id}", user_id=user.id) + log_event("session_created", user_id=user.id, service_slug=service.slug, session_id=session_id, mode="single_runtime", container_id=container_id) + + audit(db, "SESSION_CREATE", f"service={service.slug} session={session_id}", user_id=user.id) + ready = wait_for_session_route(session_id) + log_event("session_route_ready", session_id=session_id, ready=ready) return RedirectResponse(url=f"/s/{session_id}/", status_code=303) - try: - container_id = create_runtime_container(service, session_id) - except Exception as exc: - logger.exception("session_container_create_failed slug=%s user_id=%s", slug, user.id) - audit(db, "SESSION_CREATE_FAILED", f"slug={slug} err={str(exc)}", user_id=user.id) - raise HTTPException(status_code=502, detail="Session runtime failed to start") - - session_obj = SessionModel( - id=session_id, - user_id=user.id, - service_id=service.id, - container_id=container_id, - status=SessionStatus.ACTIVE, - created_at=now_utc(), - last_access_at=now_utc(), - ) - db.add(session_obj) - db.commit() - - audit(db, "SESSION_CREATE", f"service={service.slug} session={session_id}", user_id=user.id) - ready = wait_for_session_route(session_id) - logger.info("session_route_ready session_id=%s ready=%s", session_id, ready) - return RedirectResponse(url=f"/s/{session_id}/", status_code=303) - @app.get("/svc/{slug}/", response_class=HTMLResponse) def service_wait_page(slug: str, request: Request, user: User = Depends(require_user), db: Session = Depends(get_db)): @@ -1985,7 +2096,23 @@ def touch_session(session_id: str, user: User = Depends(require_user), db: Sessi if not sess or sess.user_id != user.id: raise HTTPException(status_code=404, detail="Session not found") if sess.status != SessionStatus.ACTIVE: - raise HTTPException(status_code=410, detail="Session expired") + reason = session_closed_reason(sess, db) + log_event( + "session_touch_rejected", + level=logging.WARNING, + session_id=session_id, + user_id=user.id, + status=sess.status.value, + reason=reason, + ) + return JSONResponse( + status_code=410, + content={ + "ok": False, + "reason": reason, + "status": sess.status.value, + }, + ) sess.last_access_at = now_utc() db.commit() return {"ok": True} @@ -1997,9 +2124,17 @@ def close_session(session_id: str, user: User = Depends(require_user), db: Sessi if not sess or sess.user_id != user.id: raise HTTPException(status_code=404, detail="Session not found") if sess.status != SessionStatus.ACTIVE: + log_event( + "session_close_already_closed", + session_id=session_id, + user_id=user.id, + status=sess.status.value, + reason=session_closed_reason(sess, db), + ) return {"ok": True, "status": sess.status.value} terminate_session_record(db, sess, SessionStatus.TERMINATED, stop_container=True) db.commit() + log_event("session_closed_by_user", session_id=session_id, user_id=user.id) return {"ok": True, "status": "TERMINATED"} diff --git a/docker-compose.yml b/docker-compose.yml index 7b5babd..2ca5a5b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: api: build: context: ./app - command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "6"] + command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "18"] environment: DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} SIGNING_KEY: ${SIGNING_KEY} diff --git a/docs/LOAD_TESTING.md b/docs/LOAD_TESTING.md new file mode 100644 index 0000000..e51eb17 --- /dev/null +++ b/docs/LOAD_TESTING.md @@ -0,0 +1,70 @@ +# Load Testing (k6) + +Готовый скрипт: `scripts/load/portal_k6.js`. + +## 1) Установка k6 + +Ubuntu/Debian: + +```bash +sudo gpg -k +sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 +echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list +sudo apt update && sudo apt install -y k6 +``` + +## 2) Smoke тест (быстрая проверка) + +```bash +k6 run scripts/load/portal_k6.js \ + -e BASE_URL=https://stend.4mont.ru \ + -e USERNAME=ruslan \ + -e PASSWORD='YOUR_PASSWORD' \ + -e SERVICE_SLUGS=vmmanager,termidesk \ + -e PROFILE=smoke +``` + +## 3) Нормальная нагрузка + +```bash +k6 run scripts/load/portal_k6.js \ + -e BASE_URL=https://stend.4mont.ru \ + -e USERS_CSV='user1:pass1;user2:pass2;user3:pass3' \ + -e SERVICE_SLUGS=vmmanager,termidesk,sayt-mont5 \ + -e PROFILE=load \ + -e HEARTBEATS=3 \ + -e THINK_TIME=1 +``` + +## 4) Стресс + +```bash +k6 run scripts/load/portal_k6.js \ + -e BASE_URL=https://stend.4mont.ru \ + -e USERS_CSV='user1:pass1;user2:pass2;user3:pass3;user4:pass4' \ + -e SERVICE_SLUGS=vmmanager,termidesk,sayt-mont5 \ + -e PROFILE=stress +``` + +## 5) Ключевые параметры + +- `BASE_URL` — адрес портала. +- `USERNAME`/`PASSWORD` — одиночный пользователь. +- `USERS_CSV` — набор пользователей `u1:p1;u2:p2`. +- `SERVICE_SLUGS` — список slug через запятую. +- `PROFILE` — `smoke|load|stress`. +- `HEARTBEATS` — сколько `touch` в итерации. +- `THINK_TIME` — пауза между итерациями. +- `CLOSE_SESSION=0` — не закрывать сессию в конце итерации. + +## 6) Что смотреть в результате + +- `http_req_duration` (`p95/p99`) +- `http_req_failed` +- `flow_errors` +- `open_success`, `open_rejected`, `limit_redirects`, `touch_rejected` + +## 7) Важное + +- Запускайте сначала `smoke`, потом `load`, и только затем `stress`. +- Для реалистичного теста используйте несколько пользователей, иначе упретесь в лимиты по сессиям одного пользователя. diff --git a/kiosk/entrypoint.sh b/kiosk/entrypoint.sh index 93ac39f..0418280 100755 --- a/kiosk/entrypoint.sh +++ b/kiosk/entrypoint.sh @@ -59,22 +59,31 @@ cat > /opt/portal/index.html < s.trim()) + .filter(Boolean); +const HEARTBEATS = Number(__ENV.HEARTBEATS || 3); +const THINK_TIME = Number(__ENV.THINK_TIME || 1); +const CLOSE_SESSION = (__ENV.CLOSE_SESSION || '1') !== '0'; +const PROFILE = (__ENV.PROFILE || 'smoke').toLowerCase(); + +const usersFromCsv = parseUsersCsv(__ENV.USERS_CSV || ''); +const singleUser = { + username: __ENV.USERNAME || '', + password: __ENV.PASSWORD || '', +}; + +const openAttempts = new Counter('open_attempts'); +const openSuccess = new Counter('open_success'); +const openRejected = new Counter('open_rejected'); +const limitRedirects = new Counter('limit_redirects'); +const touchRejected = new Counter('touch_rejected'); +const closeCalls = new Counter('close_calls'); +const loginFailures = new Counter('login_failures'); +const flowErrors = new Rate('flow_errors'); +const openLatency = new Trend('open_latency_ms'); + +const PROFILE_OPTIONS = { + smoke: { + vus: 5, + duration: '1m', + }, + load: { + vus: 30, + duration: '10m', + }, + stress: { + stages: [ + { duration: '2m', target: 30 }, + { duration: '5m', target: 80 }, + { duration: '2m', target: 0 }, + ], + }, +}; + +const selected = PROFILE_OPTIONS[PROFILE] || PROFILE_OPTIONS.smoke; + +export const options = { + ...selected, + insecureSkipTLSVerify: (__ENV.INSECURE_TLS || '0') === '1', + thresholds: { + http_req_failed: ['rate<0.02'], + http_req_duration: ['p(95)<2000'], + flow_errors: ['rate<0.05'], + }, +}; + +let loggedInKey = ''; + +function parseUsersCsv(raw) { + if (!raw.trim()) return []; + return raw + .split(';') + .map((pair) => pair.trim()) + .filter(Boolean) + .map((pair) => { + const idx = pair.indexOf(':'); + if (idx <= 0) return null; + return { + username: pair.slice(0, idx).trim(), + password: pair.slice(idx + 1).trim(), + }; + }) + .filter((u) => u && u.username && u.password); +} + +function currentCredentials() { + if (usersFromCsv.length > 0) { + const vu = exec.vu.idInTest || 1; + return usersFromCsv[(vu - 1) % usersFromCsv.length]; + } + return singleUser; +} + +function ensureLoggedIn() { + const creds = currentCredentials(); + if (!creds.username || !creds.password) { + fail('Set USERNAME/PASSWORD or USERS_CSV (username:password;username2:password2)'); + } + + const userKey = `${creds.username}:${creds.password}`; + if (loggedInKey === userKey) return creds; + + const jar = http.cookieJar(); + jar.clear(BASE_URL); + + const landing = http.get(`${BASE_URL}/`, { redirects: 0, tags: { name: 'GET /' } }); + const csrfFromCookie = (jar.cookiesForURL(BASE_URL).csrf_token || [])[0] || ''; + const csrf = csrfFromCookie || ((landing.cookies.csrf_token || [])[0] || {}).value || ''; + if (!csrf) { + loginFailures.add(1); + fail('CSRF cookie not found on GET /'); + } + + const payload = [ + `username=${encodeURIComponent(creds.username)}`, + `password=${encodeURIComponent(creds.password)}`, + `csrf_token=${encodeURIComponent(csrf)}`, + ].join('&'); + + const loginRes = http.post(`${BASE_URL}/login`, payload, { + redirects: 0, + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + tags: { name: 'POST /login' }, + }); + + const ok = check(loginRes, { + 'login status is redirect': (r) => r.status === 303, + }); + + const hasAuth = (jar.cookiesForURL(BASE_URL).portal_auth || []).length > 0; + if (!ok || !hasAuth) { + loginFailures.add(1); + fail(`Login failed for ${creds.username}: status=${loginRes.status}`); + } + + loggedInKey = userKey; + return creds; +} + +function pickSlug() { + return SERVICE_SLUGS[Math.floor(Math.random() * SERVICE_SLUGS.length)]; +} + +function extractSessionId(location) { + const match = (location || '').match(/\/s\/([0-9a-fA-F-]{36})\//); + return match ? match[1] : ''; +} + +export default function () { + ensureLoggedIn(); + + const slug = pickSlug(); + openAttempts.add(1); + + const openRes = http.get(`${BASE_URL}/go/${slug}`, { + redirects: 0, + tags: { name: 'GET /go/:slug' }, + }); + openLatency.add(openRes.timings.duration); + + const location = openRes.headers.Location || ''; + const sessionId = extractSessionId(location); + + const opened = check(openRes, { + 'open returns redirect': (r) => r.status === 303, + }); + + if (!opened) { + openRejected.add(1); + flowErrors.add(1); + sleep(THINK_TIME); + return; + } + + if (location.includes('launch_error=max_services')) { + limitRedirects.add(1); + flowErrors.add(1); + sleep(THINK_TIME); + return; + } + + if (!sessionId) { + openRejected.add(1); + flowErrors.add(1); + sleep(THINK_TIME); + return; + } + + openSuccess.add(1); + + for (let i = 0; i < HEARTBEATS; i += 1) { + const touchRes = http.post(`${BASE_URL}/api/sessions/${sessionId}/touch`, null, { + redirects: 0, + tags: { name: 'POST /api/sessions/:id/touch' }, + }); + + if (touchRes.status === 410) { + touchRejected.add(1); + break; + } + + const touchOk = check(touchRes, { + 'touch status 200': (r) => r.status === 200, + }); + if (!touchOk) { + flowErrors.add(1); + break; + } + sleep(0.5); + } + + if (CLOSE_SESSION) { + const closeRes = http.post(`${BASE_URL}/api/sessions/${sessionId}/close`, null, { + redirects: 0, + tags: { name: 'POST /api/sessions/:id/close' }, + }); + closeCalls.add(1); + check(closeRes, { + 'close status 200': (r) => r.status === 200, + }); + } + + flowErrors.add(0); + sleep(THINK_TIME); +} diff --git a/universal-runtime/entrypoint.sh b/universal-runtime/entrypoint.sh index dff49c8..40577d0 100755 --- a/universal-runtime/entrypoint.sh +++ b/universal-runtime/entrypoint.sh @@ -65,8 +65,15 @@ cat > /opt/portal/index.html <<'HTML' const statusEl = document.getElementById('status'); const XK_ALT_L = 0xffe9; const XK_LEFT = 0xff51; + + let rfb = null; let connected = false; let connectTimer = null; + let reconnectTimer = null; + let reconnectAttempts = 0; + const MAX_RECONNECT_ATTEMPTS = 12; + const RECONNECT_DELAYS_MS = [1000, 2000, 3000, 5000, 8000]; + let manualDisconnect = false; function showStatus(text, isError = false) { statusEl.textContent = text; @@ -78,73 +85,160 @@ cat > /opt/portal/index.html <<'HTML' statusEl.classList.add('hidden'); } - showStatus('Подключение к слоту...'); - connectTimer = setTimeout(() => { - if (!connected) { - showStatus('Нет подключения к экрану слота. Откройте сервис заново из дашборда.', true); + function clearConnectTimer() { + if (connectTimer) { + clearTimeout(connectTimer); + connectTimer = null; } - }, 8000); + } - const rfb = new RFB(document.getElementById('screen'), wsUrl); - rfb.viewOnly = false; - rfb.scaleViewport = true; - rfb.resizeSession = true; - rfb.addEventListener('connect', () => { - connected = true; - if (connectTimer) clearTimeout(connectTimer); - hideStatus(); - }); - rfb.addEventListener('disconnect', () => { + function clearReconnectTimer() { + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = null; + } + } + + function reconnectDelay(attemptNumber) { + const idx = Math.min(Math.max(attemptNumber - 1, 0), RECONNECT_DELAYS_MS.length - 1); + return RECONNECT_DELAYS_MS[idx]; + } + + function scheduleConnectTimeout() { + clearConnectTimer(); + connectTimer = setTimeout(() => { + if (!connected && !manualDisconnect) { + scheduleReconnect('Нет подключения к экрану слота. Пробуем переподключиться...'); + } + }, 8000); + } + + function scheduleReconnect(reasonText) { + if (manualDisconnect) return; + clearConnectTimer(); + clearReconnectTimer(); + + if (reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) { + showStatus('Соединение со слотом потеряно. Переподключение не удалось. Откройте сервис заново.', true); + return; + } + + const nextAttempt = reconnectAttempts + 1; + const delayMs = reconnectDelay(nextAttempt); + const delaySec = Math.ceil(delayMs / 1000); + showStatus(`${reasonText} Повтор ${nextAttempt}/${MAX_RECONNECT_ATTEMPTS} через ${delaySec} сек.`, true); + + reconnectTimer = setTimeout(() => { + reconnectAttempts = nextAttempt; + connectRfb('Переподключение к слоту...'); + }, delayMs); + } + + function attachRfb() { + if (rfb) { + try { rfb.disconnect(); } catch (e) {} + } + rfb = new RFB(document.getElementById('screen'), wsUrl); + rfb.viewOnly = false; + rfb.scaleViewport = true; + rfb.resizeSession = true; + + rfb.addEventListener('connect', () => { + connected = true; + reconnectAttempts = 0; + clearConnectTimer(); + clearReconnectTimer(); + hideStatus(); + }); + + rfb.addEventListener('disconnect', () => { + connected = false; + if (manualDisconnect) return; + scheduleReconnect('Соединение со слотом потеряно.'); + }); + } + + function connectRfb(statusText) { + if (manualDisconnect) return; connected = false; - showStatus('Соединение со слотом потеряно. Запустите сервис заново.', true); - }); + showStatus(statusText || 'Подключение к слоту...'); + attachRfb(); + scheduleConnectTimeout(); + } const enableHeartbeat = (new URLSearchParams(location.search).get('hb') ?? '1') !== '0'; const sid = new URLSearchParams(location.search).get('sid'); - const SESSION_CLOSED_URL = '/?session_closed=idle'; + const SESSION_CLOSED_URL_BASE = '/?session_closed='; const CLOSE_PATH = sid ? `/api/sessions/${sid}/close` : ''; - function goSessionClosed() { + + function goSessionClosed(reason = 'idle') { + const safeReason = reason === 'limit' ? 'limit' : 'idle'; + const target = `${SESSION_CLOSED_URL_BASE}${safeReason}`; try { if (window.top && window.top !== window) { - window.top.location.href = SESSION_CLOSED_URL; + window.top.location.href = target; return; } } catch (e) {} - window.location.href = SESSION_CLOSED_URL; + window.location.href = target; } + async function touch() { if (!sid) return; try { const res = await fetch(`/api/sessions/${sid}/touch`, {method:'POST', credentials:'include'}); if (!res.ok) { - goSessionClosed(); + let reason = 'idle'; + try { + const payload = await res.json(); + if (payload && typeof payload.reason === 'string') { + reason = payload.reason; + } + } catch (e) {} + goSessionClosed(reason); } } catch (e) {} } + let closing = false; async function closeSessionNow() { if (!CLOSE_PATH || closing) return; closing = true; + manualDisconnect = true; + clearConnectTimer(); + clearReconnectTimer(); + try { + if (rfb) rfb.disconnect(); + } catch (e) {} try { await fetch(CLOSE_PATH, {method: 'POST', credentials: 'include', keepalive: true}); } catch (e) {} } + if (enableHeartbeat) { setInterval(touch, 15000); touch(); window.addEventListener('pagehide', closeSessionNow); window.addEventListener('beforeunload', closeSessionNow); } + function keyTap(keysym, code) { + if (!rfb) return; rfb.sendKey(keysym, code, true); rfb.sendKey(keysym, code, false); } + function chord(mod, key, modCode, keyCode) { + if (!rfb) return; rfb.sendKey(mod, modCode, true); keyTap(key, keyCode); rfb.sendKey(mod, modCode, false); } + function goHome() { + manualDisconnect = true; + clearConnectTimer(); + clearReconnectTimer(); try { if (window.top && window.top !== window) { window.top.location.href = '/'; @@ -153,9 +247,12 @@ cat > /opt/portal/index.html <<'HTML' } catch (e) {} window.location.href = '/'; } + document.getElementById('btn-back').addEventListener('click', () => chord(XK_ALT_L, XK_LEFT, 'AltLeft', 'ArrowLeft')); document.getElementById('btn-home').addEventListener('click', goHome); document.addEventListener('contextmenu', (e) => e.preventDefault()); + + connectRfb('Подключение к слоту...');