Refactor code structure and optimize performance across multiple modules
deploy / deploy (push) Has been cancelled
deploy / deploy (push) Has been cancelled
This commit is contained in:
didar
@@ -1,59 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
SOURCE_CONTAINER="${SOURCE_CONTAINER:-konturai-parser-app}"
|
||||
DEPLOY_DIR="${DEPLOY_DIR:-/home/gitlab-runner/deploy/marketing-parser}"
|
||||
|
||||
ENV_KEYS=(
|
||||
SPRING_DATASOURCE_URL
|
||||
POSTGRES_USER
|
||||
POSTGRES_PASSWORD
|
||||
SPRING_JPA_HIBERNATE_DDL_AUTO
|
||||
CORS_ALLOWED_ORIGIN
|
||||
FACEBOOK_LEADS_PAGE_ID
|
||||
FACEBOOK_LEADS_ACCESS_TOKEN
|
||||
FACEBOOK_LEADS_CRON
|
||||
FACEBOOK_LEADS_GRAPH_API_VERSION
|
||||
FACEBOOK_LEADS_CALL_CENTER_BASE_URL
|
||||
)
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "Run this script as root so it can write the deploy directory for gitlab-runner." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "Docker is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! id gitlab-runner >/dev/null 2>&1; then
|
||||
echo "gitlab-runner user does not exist." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! docker inspect "$SOURCE_CONTAINER" >/dev/null 2>&1; then
|
||||
echo "Source container not found: $SOURCE_CONTAINER" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install -d -o gitlab-runner -g gitlab-runner "$DEPLOY_DIR"
|
||||
|
||||
TMP_ENV="$(mktemp)"
|
||||
trap 'rm -f "$TMP_ENV"' EXIT
|
||||
|
||||
PATTERN="$(printf '%s|' "${ENV_KEYS[@]}")"
|
||||
PATTERN="^(${PATTERN%|})="
|
||||
|
||||
docker inspect "$SOURCE_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' \
|
||||
| grep -E "$PATTERN" > "$TMP_ENV"
|
||||
|
||||
if [[ ! -s "$TMP_ENV" ]]; then
|
||||
echo "Failed to extract deployment environment from $SOURCE_CONTAINER" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install -m 600 -o gitlab-runner -g gitlab-runner "$TMP_ENV" "$DEPLOY_DIR/.env.production"
|
||||
|
||||
echo "Bootstrap complete: wrote $DEPLOY_DIR/.env.production from $SOURCE_CONTAINER"
|
||||
@@ -1,118 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DEPLOY_DIR="${DEPLOY_DIR:-/home/gitlab-runner/deploy/marketing-parser}"
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-$DEPLOY_DIR/deployment/docker-compose.server.yml}"
|
||||
PARSER_IMAGE_NAME="${PARSER_IMAGE_NAME:-marketing-parser-app}"
|
||||
PARSER_IMAGE_TAG="${PARSER_IMAGE_TAG:-${CI_COMMIT_SHORT_SHA:-latest}}"
|
||||
PARSER_IMAGE="${PARSER_IMAGE:-${PARSER_IMAGE_NAME}:${PARSER_IMAGE_TAG}}"
|
||||
PARSER_CONTAINER_NAME="${PARSER_CONTAINER_NAME:-konturai-parser-app}"
|
||||
COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-marketing-parser}"
|
||||
WAIT_TIMEOUT_SECONDS="${WAIT_TIMEOUT_SECONDS:-120}"
|
||||
|
||||
require_command() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "Missing required command: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_container() {
|
||||
local deadline status
|
||||
deadline=$((SECONDS + WAIT_TIMEOUT_SECONDS))
|
||||
|
||||
while (( SECONDS < deadline )); do
|
||||
status="$(docker inspect "$PARSER_CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || true)"
|
||||
if [[ "$status" == "running" ]]; then
|
||||
echo "Container is running: $PARSER_CONTAINER_NAME"
|
||||
return 0
|
||||
fi
|
||||
if [[ "$status" == "exited" || "$status" == "dead" ]]; then
|
||||
echo "Container $PARSER_CONTAINER_NAME is in state: $status" >&2
|
||||
docker logs --tail 100 "$PARSER_CONTAINER_NAME" || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
echo "Timed out waiting for $PARSER_CONTAINER_NAME to start" >&2
|
||||
docker logs --tail 100 "$PARSER_CONTAINER_NAME" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_command bash
|
||||
require_command docker
|
||||
require_command rsync
|
||||
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
COMPOSE_CMD=(docker compose)
|
||||
elif command -v docker-compose >/dev/null 2>&1; then
|
||||
COMPOSE_CMD=(docker-compose)
|
||||
else
|
||||
echo "Missing docker compose or docker-compose" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${CI_PROJECT_DIR:-}" || ! -d "${CI_PROJECT_DIR:-}" ]]; then
|
||||
echo "CI_PROJECT_DIR is not set or does not exist" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
echo "Docker daemon is not reachable for the runner user" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install -d "$DEPLOY_DIR"
|
||||
|
||||
if [[ "$CI_PROJECT_DIR" != "$DEPLOY_DIR" ]]; then
|
||||
rsync -a --delete \
|
||||
--exclude '.git/' \
|
||||
--exclude '.m2/' \
|
||||
--exclude 'target/' \
|
||||
--exclude '.codex' \
|
||||
--exclude '.claude/' \
|
||||
--exclude '.cursor/' \
|
||||
--exclude '.idea/' \
|
||||
--exclude '.vscode/' \
|
||||
--exclude '.DS_Store' \
|
||||
--exclude '.env' \
|
||||
--exclude '.env.*' \
|
||||
"$CI_PROJECT_DIR/" "$DEPLOY_DIR/"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$DEPLOY_DIR/.env.production" ]]; then
|
||||
if [[ -n "${DEPLOY_ENV_FILE:-}" && -f "${DEPLOY_ENV_FILE}" ]]; then
|
||||
install -m 600 "$DEPLOY_ENV_FILE" "$DEPLOY_DIR/.env.production"
|
||||
else
|
||||
echo "Missing $DEPLOY_DIR/.env.production. Run scripts/bootstrap_gitlab_deploy.sh once or set DEPLOY_ENV_FILE." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if docker ps -a --format '{{.Names}}' | grep -Fxq "$PARSER_CONTAINER_NAME"; then
|
||||
EXISTING_PROJECT="$(docker inspect "$PARSER_CONTAINER_NAME" --format '{{ index .Config.Labels "com.docker.compose.project" }}' 2>/dev/null || true)"
|
||||
if [[ -n "$EXISTING_PROJECT" && "$EXISTING_PROJECT" != "$COMPOSE_PROJECT_NAME" ]]; then
|
||||
echo "Removing existing container $PARSER_CONTAINER_NAME from compose project $EXISTING_PROJECT"
|
||||
docker rm -f "$PARSER_CONTAINER_NAME"
|
||||
fi
|
||||
fi
|
||||
|
||||
cd "$DEPLOY_DIR"
|
||||
|
||||
echo "Deploy directory: $DEPLOY_DIR"
|
||||
echo "Building image: $PARSER_IMAGE"
|
||||
|
||||
docker build -t "$PARSER_IMAGE" -t "${PARSER_IMAGE_NAME}:latest" "$DEPLOY_DIR"
|
||||
|
||||
export PARSER_IMAGE
|
||||
export PARSER_CONTAINER_NAME
|
||||
export COMPOSE_PROJECT_NAME
|
||||
|
||||
"${COMPOSE_CMD[@]}" -f "$COMPOSE_FILE" config -q
|
||||
"${COMPOSE_CMD[@]}" -f "$COMPOSE_FILE" up -d --remove-orphans --force-recreate
|
||||
|
||||
wait_for_container
|
||||
|
||||
docker ps --filter "name=^${PARSER_CONTAINER_NAME}$" --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}'
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
GITLAB_URL="${GITLAB_URL:-https://gitlab.konturai.kz/}"
|
||||
RUNNER_NAME="${RUNNER_NAME:-marketing-parser-prod-runner}"
|
||||
RUNNER_TAGS="${RUNNER_TAGS:-marketing-parser-prod}"
|
||||
RUNNER_EXECUTOR="${RUNNER_EXECUTOR:-shell}"
|
||||
RUNNER_TOKEN="${RUNNER_TOKEN:-}"
|
||||
DEPLOY_DIR="${DEPLOY_DIR:-/home/gitlab-runner/deploy/marketing-parser}"
|
||||
SKIP_REGISTER="${SKIP_REGISTER:-0}"
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "Run this script as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$SKIP_REGISTER" != "1" && -z "$RUNNER_TOKEN" ]]; then
|
||||
echo "Set RUNNER_TOKEN from GitLab project/group runner settings before running this script." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl rsync docker.io docker-compose-plugin
|
||||
|
||||
if ! command -v gitlab-runner >/dev/null 2>&1; then
|
||||
curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | bash
|
||||
apt-get install -y gitlab-runner
|
||||
fi
|
||||
|
||||
if ! getent group docker >/dev/null 2>&1; then
|
||||
echo "Docker group does not exist. Install Docker before registering the runner." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
usermod -aG docker gitlab-runner
|
||||
install -d -o gitlab-runner -g gitlab-runner "$(dirname "$DEPLOY_DIR")"
|
||||
install -d -o gitlab-runner -g gitlab-runner "$DEPLOY_DIR"
|
||||
|
||||
if [[ "$SKIP_REGISTER" == "1" ]]; then
|
||||
echo "gitlab-runner installed. Registration skipped because SKIP_REGISTER=1."
|
||||
elif gitlab-runner list 2>/dev/null | grep -Fq "$RUNNER_NAME"; then
|
||||
echo "Runner $RUNNER_NAME is already registered."
|
||||
else
|
||||
register_args=(
|
||||
--non-interactive
|
||||
--url "$GITLAB_URL"
|
||||
--executor "$RUNNER_EXECUTOR"
|
||||
--description "$RUNNER_NAME"
|
||||
--tag-list "$RUNNER_TAGS"
|
||||
--run-untagged="false"
|
||||
--locked="true"
|
||||
)
|
||||
if [[ "$RUNNER_TOKEN" == glrt-* ]]; then
|
||||
register_args+=(--token "$RUNNER_TOKEN")
|
||||
else
|
||||
register_args+=(--registration-token "$RUNNER_TOKEN")
|
||||
fi
|
||||
gitlab-runner register "${register_args[@]}"
|
||||
fi
|
||||
|
||||
systemctl enable --now gitlab-runner
|
||||
systemctl restart gitlab-runner
|
||||
if [[ "$SKIP_REGISTER" != "1" ]]; then
|
||||
gitlab-runner verify
|
||||
fi
|
||||
|
||||
if [[ "$SKIP_REGISTER" == "1" ]]; then
|
||||
echo "gitlab-runner service is installed and running."
|
||||
else
|
||||
echo "Runner $RUNNER_NAME is ready."
|
||||
fi
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
|
||||
echo "🚀 Запуск RSS Parser для Kursiv Media"
|
||||
echo "======================================"
|
||||
|
||||
# Проверяем наличие Java
|
||||
if ! command -v java &> /dev/null; then
|
||||
echo "❌ Java не найдена. Убедитесь, что Java 21+ установлена."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Проверяем версию Java
|
||||
JAVA_VERSION=$(java -version 2>&1 | head -n 1 | cut -d'"' -f2 | cut -d'.' -f1)
|
||||
if [ "$JAVA_VERSION" -lt 21 ]; then
|
||||
echo "❌ Требуется Java 21 или выше. Текущая версия: $JAVA_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Java версия: $(java -version 2>&1 | head -n 1)"
|
||||
|
||||
# Проверяем наличие Maven wrapper
|
||||
if [ ! -f "./mvnw" ]; then
|
||||
echo "❌ Maven wrapper не найден"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Maven wrapper найден"
|
||||
|
||||
# Собираем проект
|
||||
echo "📦 Сборка проекта..."
|
||||
./mvnw clean compile -q
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "❌ Ошибка при сборке проекта"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Проект собран успешно"
|
||||
|
||||
# Проверяем доступность MongoDB (опционально)
|
||||
echo "🔍 Проверка MongoDB..."
|
||||
if command -v mongosh &> /dev/null; then
|
||||
if mongosh --eval "db.runCommand('ping')" --quiet &> /dev/null; then
|
||||
echo "✅ MongoDB доступен"
|
||||
else
|
||||
echo "⚠️ MongoDB может быть недоступен. Убедитесь, что MongoDB запущен."
|
||||
fi
|
||||
else
|
||||
echo "⚠️ MongoDB клиент не найден. Убедитесь, что MongoDB установлен и запущен."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "🎯 Доступные команды:"
|
||||
echo "1. Запуск приложения: ./mvnw spring-boot:run"
|
||||
echo "2. Запуск тестов: ./mvnw test"
|
||||
echo "3. Сборка JAR: ./mvnw clean package"
|
||||
echo ""
|
||||
echo "📡 После запуска приложения API будет доступно по адресу:"
|
||||
echo " http://localhost:8080/api/parser/"
|
||||
echo ""
|
||||
echo "🔧 Основные endpoints:"
|
||||
echo " POST /api/parser/parse - Запуск парсинга"
|
||||
echo " GET /api/parser/stats - Статистика"
|
||||
echo " GET /api/parser/items - Все записи"
|
||||
echo " GET /api/parser/health - Проверка состояния"
|
||||
echo ""
|
||||
|
||||
# Спрашиваем, хочет ли пользователь запустить приложение
|
||||
read -p "🚀 Запустить приложение сейчас? (y/n): " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "🚀 Запуск приложения..."
|
||||
./mvnw spring-boot:run
|
||||
else
|
||||
echo "👋 Для запуска используйте: ./mvnw spring-boot:run"
|
||||
fi
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
# Checks that the configured MongoDB server is reachable and prints a ready-to-use
|
||||
# connection string. Reads the same environment variables the application uses.
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${MONGODB_HOST:-localhost}"
|
||||
PORT="${MONGODB_PORT:-27017}"
|
||||
DATABASE="${MONGODB_DATABASE:-parser_db}"
|
||||
USERNAME="${MONGODB_USERNAME:-}"
|
||||
AUTH_DB="${MONGODB_AUTH_DATABASE:-admin}"
|
||||
|
||||
echo "🔍 Checking MongoDB connectivity..."
|
||||
echo "======================================"
|
||||
echo "📡 Probing ${HOST}:${PORT}..."
|
||||
|
||||
if nc -z "$HOST" "$PORT"; then
|
||||
echo "✅ MongoDB server is reachable"
|
||||
else
|
||||
echo "❌ MongoDB server is unreachable"
|
||||
echo " Check that the server is running and that MONGODB_HOST/MONGODB_PORT are correct."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "🔐 Configured connection:"
|
||||
echo " Host: ${HOST}:${PORT}"
|
||||
echo " Database: ${DATABASE}"
|
||||
echo " Username: ${USERNAME:-<MONGODB_USERNAME not set>}"
|
||||
echo " Auth DB: ${AUTH_DB}"
|
||||
|
||||
echo ""
|
||||
echo "📝 Verify credentials with:"
|
||||
echo " mongosh \"mongodb://${USERNAME:-<user>}@${HOST}:${PORT}/${DATABASE}?authSource=${AUTH_DB}\""
|
||||
echo " (mongosh will prompt for the password; never pass it on the command line)"
|
||||
|
||||
echo ""
|
||||
echo "⚠️ If authentication fails, check:"
|
||||
echo " 1. MONGODB_USERNAME / MONGODB_PASSWORD are exported"
|
||||
echo " 2. Firewall rules on the MongoDB host"
|
||||
echo " 3. The user's grants on '${DATABASE}'"
|
||||
echo " 4. That authSource matches how the user was created"
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Скрипт для тестирования OpenAI API ключа
|
||||
|
||||
echo "🔍 Тестирование OpenAI API ключа..."
|
||||
|
||||
# Проверяем, установлена ли переменная окружения
|
||||
if [ -z "$OPENAI_API_KEY" ]; then
|
||||
echo "❌ Переменная окружения OPENAI_API_KEY не установлена"
|
||||
echo "Установите её командой: export OPENAI_API_KEY=sk-your-api-key-here"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Проверяем формат ключа
|
||||
if [[ ! $OPENAI_API_KEY =~ ^sk- ]]; then
|
||||
echo "❌ API ключ должен начинаться с 'sk-'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ API ключ найден: ${OPENAI_API_KEY:0:10}..."
|
||||
|
||||
# Тестируем API
|
||||
echo "🚀 Отправка тестового запроса к OpenAI API..."
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "https://api.openai.com/v1/chat/completions" \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Привет! Это тест API ключа. Ответь коротко."
|
||||
}
|
||||
],
|
||||
"max_tokens": 50
|
||||
}')
|
||||
|
||||
# Извлекаем HTTP код и тело ответа
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | head -n -1)
|
||||
|
||||
echo "📊 HTTP код: $http_code"
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
echo "✅ API ключ работает корректно!"
|
||||
echo "📝 Ответ от OpenAI:"
|
||||
echo "$response_body" | jq -r '.choices[0].message.content' 2>/dev/null || echo "$response_body"
|
||||
elif [ "$http_code" = "401" ]; then
|
||||
echo "❌ Ошибка 401: API ключ недействителен или истек"
|
||||
echo "📝 Ответ: $response_body"
|
||||
elif [ "$http_code" = "429" ]; then
|
||||
echo "⚠️ Ошибка 429: Превышен лимит запросов"
|
||||
echo "📝 Ответ: $response_body"
|
||||
else
|
||||
echo "❌ Ошибка $http_code"
|
||||
echo "📝 Ответ: $response_body"
|
||||
fi
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Тестовый скрипт для проверки нового API эндпоинта /api/parser/report
|
||||
|
||||
echo "Тестирование API эндпоинта для генерации исследовательских отчётов"
|
||||
echo "=================================================================="
|
||||
|
||||
# URL API
|
||||
API_URL="http://localhost:8080/api/parser/report"
|
||||
|
||||
# Тестовые данные
|
||||
echo "Отправка запроса на генерацию отчёта..."
|
||||
|
||||
curl -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"query": "Искусственный интеллект в здравоохранении",
|
||||
"lang": "ru",
|
||||
"depth": 3,
|
||||
"breadth": 5,
|
||||
"report_type": "report"
|
||||
}' \
|
||||
--output "research_report_test.pdf" \
|
||||
--write-out "HTTP Status: %{http_code}\nResponse Time: %{time_total}s\n"
|
||||
|
||||
echo ""
|
||||
echo "Проверка статуса ответа..."
|
||||
if [ -f "research_report_test.pdf" ]; then
|
||||
echo "✅ PDF файл успешно создан: research_report_test.pdf"
|
||||
echo "Размер файла: $(ls -lh research_report_test.pdf | awk '{print $5}')"
|
||||
else
|
||||
echo "❌ PDF файл не был создан"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Тестирование с минимальными параметрами..."
|
||||
curl -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"query": "Блокчейн технологии"
|
||||
}' \
|
||||
--output "research_report_minimal.pdf" \
|
||||
--write-out "HTTP Status: %{http_code}\nResponse Time: %{time_total}s\n"
|
||||
|
||||
echo ""
|
||||
echo "Тестирование с некорректными данными (должен вернуть 400)..."
|
||||
curl -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"query": "",
|
||||
"depth": 10
|
||||
}' \
|
||||
--write-out "HTTP Status: %{http_code}\n"
|
||||
|
||||
echo ""
|
||||
echo "Тестирование завершено!"
|
||||
Reference in New Issue
Block a user