Add runner-based auto deploy for marketing-parser

2026-04-12 21:36:17 +05:00
parent 09ab38d738
commit 2c12b4940d
6 changed files with 346 additions and 0 deletions
@@ -2,9 +2,13 @@ image: eclipse-temurin:21-jdk

 variables:
  MAVEN_OPTS: "-Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository"
+  DEPLOY_DIR: "/home/gitlab-runner/deploy/marketing-parser"
+  PARSER_IMAGE_NAME: "marketing-parser-app"
+  PARSER_CONTAINER_NAME: "konturai-parser-app"

 stages:
  - test
+  - deploy

 cache:
  key: ${CI_COMMIT_REF_SLUG}
@@ -24,3 +28,29 @@ test:
    reports:
      junit:
        - target/surefire-reports/TEST-*.xml
+
+deploy_production:
+  stage: deploy
+  needs:
+    - test
+  tags:
+    - marketing-parser-prod
+  variables:
+    DOCKER_HOST: ""
+    DOCKER_TLS_VERIFY: ""
+    DOCKER_CERT_PATH: ""
+    DOCKER_TLS_CERTDIR: ""
+    COMPOSE_FILE: "$DEPLOY_DIR/deployment/docker-compose.server.yml"
+    COMPOSE_PROJECT_NAME: "marketing-parser"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - command -v bash >/dev/null
+    - command -v docker >/dev/null
+    - command -v rsync >/dev/null
+  script:
+    - bash scripts/deploy_gitlab.sh
+  environment:
+    name: production
+  resource_group: marketing-parser-production
@@ -0,0 +1,45 @@
+# Deployment
+
+This repository now uses the same GitLab shell-runner deployment pattern as `call-center`: no project CI/CD variables are required for the normal push deploy path.
+
+## What happens on push
+
+1. GitLab runs the existing `test` job.
+2. A dedicated shell runner with tag `marketing-parser-prod` picks up `deploy_production`.
+3. The runner syncs the repository into `/home/gitlab-runner/deploy/marketing-parser`.
+4. The runner builds a local Docker image from that synced checkout.
+5. Docker Compose recreates `konturai-parser-app` from [deployment/docker-compose.server.yml](/root/marketing-parser/deployment/docker-compose.server.yml:1).
+
+## Files used
+
+- [.gitlab-ci.yml](/root/marketing-parser/.gitlab-ci.yml:1)
+- [deployment/docker-compose.server.yml](/root/marketing-parser/deployment/docker-compose.server.yml:1)
+- [scripts/deploy_gitlab.sh](/root/marketing-parser/scripts/deploy_gitlab.sh:1)
+- [scripts/bootstrap_gitlab_deploy.sh](/root/marketing-parser/scripts/bootstrap_gitlab_deploy.sh:1)
+- [scripts/install_gitlab_runner.sh](/root/marketing-parser/scripts/install_gitlab_runner.sh:1)
+
+## One-time server setup
+
+Install and register a shell runner on the target server:
+
+```bash
+cd /root/marketing-parser
+RUNNER_TOKEN=glrt-xxxxxxxx bash scripts/install_gitlab_runner.sh
+```
+
+Then bootstrap the production env file from the current `konturai-parser-app` container:
+
+```bash
+cd /root/marketing-parser
+bash scripts/bootstrap_gitlab_deploy.sh
+```
+
+That writes `/home/gitlab-runner/deploy/marketing-parser/.env.production`, which is used by the deploy job.
+
+## Important notes
+
+- The deploy job runs for pushes and manual web-triggered pipelines on the branch that contains this CI configuration.
+- The runner user must have Docker access.
+- This setup expects a host `shell` runner. A minimal `gitlab/gitlab-runner:alpine` container is not enough unless you also provide Docker, Compose, and `rsync` inside that runner environment.
+- This deploy path is intentionally independent from `/root/docker-compose.yaml` because `/root` is not accessible to the `gitlab-runner` user.
+- The deployment compose file joins the existing external Docker network `common_network` and publishes the service alias `parser-service`.
@@ -0,0 +1,22 @@
+name: marketing-parser
+
+services:
+  parser-service:
+    build:
+      context: ..
+      dockerfile: Dockerfile
+    image: ${PARSER_IMAGE:-marketing-parser-app:latest}
+    container_name: ${PARSER_CONTAINER_NAME:-konturai-parser-app}
+    restart: unless-stopped
+    env_file:
+      - ../.env.production
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    networks:
+      common_network:
+        aliases:
+          - parser-service
+
+networks:
+  common_network:
+    external: true
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+SOURCE_CONTAINER="${SOURCE_CONTAINER:-konturai-parser-app}"
+DEPLOY_DIR="${DEPLOY_DIR:-/home/gitlab-runner/deploy/marketing-parser}"
+
+ENV_KEYS=(
+  SPRING_DATASOURCE_URL
+  POSTGRES_USER
+  POSTGRES_PASSWORD
+  SPRING_JPA_HIBERNATE_DDL_AUTO
+  CORS_ALLOWED_ORIGIN
+  FACEBOOK_LEADS_PAGE_ID
+  FACEBOOK_LEADS_ACCESS_TOKEN
+  FACEBOOK_LEADS_CRON
+  FACEBOOK_LEADS_GRAPH_API_VERSION
+  FACEBOOK_LEADS_CALL_CENTER_BASE_URL
+)
+
+if [[ "${EUID}" -ne 0 ]]; then
+  echo "Run this script as root so it can write the deploy directory for gitlab-runner." >&2
+  exit 1
+fi
+
+if ! command -v docker >/dev/null 2>&1; then
+  echo "Docker is required." >&2
+  exit 1
+fi
+
+if ! id gitlab-runner >/dev/null 2>&1; then
+  echo "gitlab-runner user does not exist." >&2
+  exit 1
+fi
+
+if ! docker inspect "$SOURCE_CONTAINER" >/dev/null 2>&1; then
+  echo "Source container not found: $SOURCE_CONTAINER" >&2
+  exit 1
+fi
+
+install -d -o gitlab-runner -g gitlab-runner "$DEPLOY_DIR"
+
+TMP_ENV="$(mktemp)"
+trap 'rm -f "$TMP_ENV"' EXIT
+
+PATTERN="$(printf '%s|' "${ENV_KEYS[@]}")"
+PATTERN="^(${PATTERN%|})="
+
+docker inspect "$SOURCE_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' \
+  | grep -E "$PATTERN" > "$TMP_ENV"
+
+if [[ ! -s "$TMP_ENV" ]]; then
+  echo "Failed to extract deployment environment from $SOURCE_CONTAINER" >&2
+  exit 1
+fi
+
+install -m 600 -o gitlab-runner -g gitlab-runner "$TMP_ENV" "$DEPLOY_DIR/.env.production"
+
+echo "Bootstrap complete: wrote $DEPLOY_DIR/.env.production from $SOURCE_CONTAINER"
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+DEPLOY_DIR="${DEPLOY_DIR:-/home/gitlab-runner/deploy/marketing-parser}"
+COMPOSE_FILE="${COMPOSE_FILE:-$DEPLOY_DIR/deployment/docker-compose.server.yml}"
+PARSER_IMAGE_NAME="${PARSER_IMAGE_NAME:-marketing-parser-app}"
+PARSER_IMAGE_TAG="${PARSER_IMAGE_TAG:-${CI_COMMIT_SHORT_SHA:-latest}}"
+PARSER_IMAGE="${PARSER_IMAGE:-${PARSER_IMAGE_NAME}:${PARSER_IMAGE_TAG}}"
+PARSER_CONTAINER_NAME="${PARSER_CONTAINER_NAME:-konturai-parser-app}"
+COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-marketing-parser}"
+WAIT_TIMEOUT_SECONDS="${WAIT_TIMEOUT_SECONDS:-120}"
+
+require_command() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "Missing required command: $1" >&2
+    exit 1
+  fi
+}
+
+wait_for_container() {
+  local deadline status
+  deadline=$((SECONDS + WAIT_TIMEOUT_SECONDS))
+
+  while (( SECONDS < deadline )); do
+    status="$(docker inspect "$PARSER_CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || true)"
+    if [[ "$status" == "running" ]]; then
+      echo "Container is running: $PARSER_CONTAINER_NAME"
+      return 0
+    fi
+    if [[ "$status" == "exited" || "$status" == "dead" ]]; then
+      echo "Container $PARSER_CONTAINER_NAME is in state: $status" >&2
+      docker logs --tail 100 "$PARSER_CONTAINER_NAME" || true
+      exit 1
+    fi
+    sleep 3
+  done
+
+  echo "Timed out waiting for $PARSER_CONTAINER_NAME to start" >&2
+  docker logs --tail 100 "$PARSER_CONTAINER_NAME" || true
+  exit 1
+}
+
+require_command bash
+require_command docker
+require_command rsync
+
+if docker compose version >/dev/null 2>&1; then
+  COMPOSE_CMD=(docker compose)
+elif command -v docker-compose >/dev/null 2>&1; then
+  COMPOSE_CMD=(docker-compose)
+else
+  echo "Missing docker compose or docker-compose" >&2
+  exit 1
+fi
+
+if [[ -z "${CI_PROJECT_DIR:-}" || ! -d "${CI_PROJECT_DIR:-}" ]]; then
+  echo "CI_PROJECT_DIR is not set or does not exist" >&2
+  exit 1
+fi
+
+if ! docker info >/dev/null 2>&1; then
+  echo "Docker daemon is not reachable for the runner user" >&2
+  exit 1
+fi
+
+install -d "$DEPLOY_DIR"
+
+if [[ "$CI_PROJECT_DIR" != "$DEPLOY_DIR" ]]; then
+  rsync -a --delete \
+    --exclude '.git/' \
+    --exclude '.m2/' \
+    --exclude 'target/' \
+    --exclude '.codex' \
+    --exclude '.claude/' \
+    --exclude '.cursor/' \
+    --exclude '.idea/' \
+    --exclude '.vscode/' \
+    --exclude '.DS_Store' \
+    --exclude '.env' \
+    --exclude '.env.*' \
+    "$CI_PROJECT_DIR/" "$DEPLOY_DIR/"
+fi
+
+if [[ ! -f "$DEPLOY_DIR/.env.production" ]]; then
+  if [[ -n "${DEPLOY_ENV_FILE:-}" && -f "${DEPLOY_ENV_FILE}" ]]; then
+    install -m 600 "$DEPLOY_ENV_FILE" "$DEPLOY_DIR/.env.production"
+  else
+    echo "Missing $DEPLOY_DIR/.env.production. Run scripts/bootstrap_gitlab_deploy.sh once or set DEPLOY_ENV_FILE." >&2
+    exit 1
+  fi
+fi
+
+if docker ps -a --format '{{.Names}}' | grep -Fxq "$PARSER_CONTAINER_NAME"; then
+  EXISTING_PROJECT="$(docker inspect "$PARSER_CONTAINER_NAME" --format '{{ index .Config.Labels "com.docker.compose.project" }}' 2>/dev/null || true)"
+  if [[ -n "$EXISTING_PROJECT" && "$EXISTING_PROJECT" != "$COMPOSE_PROJECT_NAME" ]]; then
+    echo "Removing existing container $PARSER_CONTAINER_NAME from compose project $EXISTING_PROJECT"
+    docker rm -f "$PARSER_CONTAINER_NAME"
+  fi
+fi
+
+cd "$DEPLOY_DIR"
+
+echo "Deploy directory: $DEPLOY_DIR"
+echo "Building image: $PARSER_IMAGE"
+
+docker build -t "$PARSER_IMAGE" -t "${PARSER_IMAGE_NAME}:latest" "$DEPLOY_DIR"
+
+export PARSER_IMAGE
+export PARSER_CONTAINER_NAME
+export COMPOSE_PROJECT_NAME
+
+"${COMPOSE_CMD[@]}" -f "$COMPOSE_FILE" config -q
+"${COMPOSE_CMD[@]}" -f "$COMPOSE_FILE" up -d --remove-orphans --force-recreate
+
+wait_for_container
+
+docker ps --filter "name=^${PARSER_CONTAINER_NAME}$" --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}'
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+GITLAB_URL="${GITLAB_URL:-https://gitlab.konturai.kz/}"
+RUNNER_NAME="${RUNNER_NAME:-marketing-parser-prod-runner}"
+RUNNER_TAGS="${RUNNER_TAGS:-marketing-parser-prod}"
+RUNNER_EXECUTOR="${RUNNER_EXECUTOR:-shell}"
+RUNNER_TOKEN="${RUNNER_TOKEN:-}"
+DEPLOY_DIR="${DEPLOY_DIR:-/home/gitlab-runner/deploy/marketing-parser}"
+SKIP_REGISTER="${SKIP_REGISTER:-0}"
+
+if [[ "${EUID}" -ne 0 ]]; then
+  echo "Run this script as root." >&2
+  exit 1
+fi
+
+if [[ "$SKIP_REGISTER" != "1" && -z "$RUNNER_TOKEN" ]]; then
+  echo "Set RUNNER_TOKEN from GitLab project/group runner settings before running this script." >&2
+  exit 1
+fi
+
+apt-get update
+apt-get install -y ca-certificates curl rsync docker.io docker-compose-plugin
+
+if ! command -v gitlab-runner >/dev/null 2>&1; then
+  curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | bash
+  apt-get install -y gitlab-runner
+fi
+
+if ! getent group docker >/dev/null 2>&1; then
+  echo "Docker group does not exist. Install Docker before registering the runner." >&2
+  exit 1
+fi
+
+usermod -aG docker gitlab-runner
+install -d -o gitlab-runner -g gitlab-runner "$(dirname "$DEPLOY_DIR")"
+install -d -o gitlab-runner -g gitlab-runner "$DEPLOY_DIR"
+
+if [[ "$SKIP_REGISTER" == "1" ]]; then
+  echo "gitlab-runner installed. Registration skipped because SKIP_REGISTER=1."
+elif gitlab-runner list 2>/dev/null | grep -Fq "$RUNNER_NAME"; then
+  echo "Runner $RUNNER_NAME is already registered."
+else
+  register_args=(
+    --non-interactive
+    --url "$GITLAB_URL"
+    --executor "$RUNNER_EXECUTOR"
+    --description "$RUNNER_NAME"
+    --tag-list "$RUNNER_TAGS"
+    --run-untagged="false"
+    --locked="true"
+  )
+  if [[ "$RUNNER_TOKEN" == glrt-* ]]; then
+    register_args+=(--token "$RUNNER_TOKEN")
+  else
+    register_args+=(--registration-token "$RUNNER_TOKEN")
+  fi
+  gitlab-runner register "${register_args[@]}"
+fi
+
+systemctl enable --now gitlab-runner
+systemctl restart gitlab-runner
+if [[ "$SKIP_REGISTER" != "1" ]]; then
+  gitlab-runner verify
+fi
+
+if [[ "$SKIP_REGISTER" == "1" ]]; then
+  echo "gitlab-runner service is installed and running."
+else
+  echo "Runner $RUNNER_NAME is ready."
+fi