digiops/marketing-parser
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
root
2026-01-14 20:28:54 +05:00
parent 6da379c25e
commit 463a25a15d
2 changed files with 299 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/kz/konturai/parser/controller/MarketingController.java
+6 -4
View File
@@ -94,11 +94,13 @@ public class MarketingController {
// errors.
boolean legacyGenerateDisabled = true;
if (legacyGenerateDisabled) {
ErrorResponse disabledError = new ErrorResponse(
"ENDPOINT_DISABLED",
MarketingAnalysisResponse disabledResponse = new MarketingAnalysisResponse(
null,
"disabled",
null,
"Обычный generate временно отключен. Используйте V2: POST /api/marketing/analysis/start/v2");
return ResponseEntity.status(HttpStatus.GONE)
.body(ApiResponse.error("Обычный generate временно отключен", disabledError));
return ResponseEntity.ok(ApiResponse.success("Обычный generate временно отключен",
disabledResponse));
}
// Create unified analysis with all analysis types
src/main/java/kz/konturai/parser/service/MarketingAnalysisService.java
+293 -15
View File
@@ -21,6 +21,7 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.LocalDateTime;
@@ -649,7 +650,10 @@ public class MarketingAnalysisService {
Map<String, Object> researchPack = buildResearchPack(request);
// 2) Build strict schema instructions + provide an example JSON structure
String exampleJson = objectMapper.writeValueAsString(buildAnalysisV2ExampleForPrompt(request));
List<String> competitorCompanies = resolveCompetitorCompaniesForBenchmarks(request, researchPack, 5);
researchPack.put("competitor_companies", competitorCompanies);
String exampleJson = objectMapper.writeValueAsString(
buildAnalysisV2ExampleForPrompt(request, competitorCompanies));
String systemPrompt = ""
+ "Ты — партнер Big Four (управленческий консалтинг) и senior marketing analyst.\n"
@@ -664,7 +668,7 @@ public class MarketingAnalysisService {
List<String> ageRanges = (List<String>) targetAudienceData.get("ageRanges");
if (ageRanges != null && !ageRanges.isEmpty()) {
ageRangesInstruction = "\n"
+ "7) КРИТИЧЕСКИ ВАЖНО для target_audience.age_structure: Используй ТОЛЬКО те возрастные группы, которые указаны в запросе (request.targetAudience.ageRanges). "
+ "8) КРИТИЧЕСКИ ВАЖНО для target_audience.age_structure: Используй ТОЛЬКО те возрастные группы, которые указаны в запросе (request.targetAudience.ageRanges). "
+ "Указанные возрастные группы: " + String.join(", ", ageRanges) + ". "
+ "НЕ добавляй возрастные группы, которых нет в запросе. "
+ "Если в запросе указаны только \"25-34\" и \"35-44\", то в age_structure должны быть ТОЛЬКО эти две группы.\n";
@@ -685,7 +689,10 @@ public class MarketingAnalysisService {
+ " - funnel_metrics.leads/bookings/payments => integer\n"
+ " - market_overview.geography => object: ключ=город, значение=целое число (индекс интереса, 0..100)\n"
+ "5) Пиши на русском языке.\n"
+ "6) НИКОГДА не копируй значения из примера. В примере намеренно стоят плейсхолдеры, "
+ "6) competitive_environment.digital_visibility_benchmarks и competitive_environment.competitor_strengths: "
+ "ключи должны быть названиями РЕАЛЬНЫХ компаний/брендов (используй evidence.competitor_companies и evidence.followups.competitors). "
+ "Запрещены ключи-заглушки: competitor_a/competitor_b/competitor_c/your_business/lead/payment/others.\n"
+ "7) НИКОГДА не копируй значения из примера. В примере намеренно стоят плейсхолдеры, "
+ "которые начинаются с 'EXAMPLE__' и числа -9999/-9999.0 — в твоем ответе их быть НЕ должно."
+ ageRangesInstruction
+ "\nНиже пример правильной структуры (пример только для формы/полей; все значения должны быть заменены на значения из evidence):\n"
@@ -742,7 +749,9 @@ public class MarketingAnalysisService {
continue;
}
return objectMapper.readValue(cleaned, MarketingAnalysisResponseV2.class);
MarketingAnalysisResponseV2 parsed = objectMapper.readValue(cleaned, MarketingAnalysisResponseV2.class);
replacePlaceholderCompanyKeysIfNeeded(parsed, competitorCompanies);
return parsed;
}
throw new IllegalStateException(
@@ -762,7 +771,273 @@ public class MarketingAnalysisService {
return true;
}
// Numeric placeholders from buildAnalysisV2ExampleForPrompt()
return json.contains("-9999");
if (json.contains("-9999")) {
return true;
}
// Check placeholder competitor keys specifically inside competitive_environment
// maps.
// This avoids false positives for normal Russian words like "лид/оплата"
// appearing in conclusions.
try {
JsonNode root = objectMapper.readTree(json);
JsonNode ce = root.at("/sections/competitive_environment");
if (ce == null || ce.isMissingNode()) {
return false;
}
JsonNode dvb = ce.get("digital_visibility_benchmarks");
if (dvb != null && dvb.isObject()) {
Iterator<String> it = dvb.fieldNames();
while (it.hasNext()) {
if (isPlaceholderCompetitorKey(it.next())) {
return true;
}
}
}
JsonNode strengths = ce.get("competitor_strengths");
if (strengths != null && strengths.isObject()) {
Iterator<String> it = strengths.fieldNames();
while (it.hasNext()) {
if (isPlaceholderCompetitorKey(it.next())) {
return true;
}
}
}
return false;
} catch (Exception e) {
// If we can't parse JSON here, treat as unusable and retry.
return true;
}
}
private boolean isPlaceholderCompetitorKey(String key) {
if (key == null) {
return false;
}
String k = key.trim().toLowerCase(Locale.ROOT);
if (k.isEmpty()) {
return false;
}
return k.startsWith("competitor_")
|| "your_business".equals(k)
|| "lead".equals(k)
|| "payment".equals(k)
|| "others".equals(k);
}
private List<String> resolveCompetitorCompaniesForBenchmarks(MarketingAnalysisRequest request,
Map<String, Object> researchPack,
int limit) {
int max = Math.max(1, Math.min(limit, 10));
LinkedHashSet<String> names = new LinkedHashSet<>();
SerperSearchResult fromPack = null;
if (researchPack != null) {
Object followupsObj = researchPack.get("followups");
if (followupsObj instanceof Map<?, ?> followups) {
Object compObj = followups.get("competitors");
if (compObj instanceof SerperSearchResult sr) {
fromPack = sr;
}
}
}
names.addAll(extractCompanyNamesFromSerper(fromPack, max));
// If competitors query returned mostly directories/articles, do a second query
// to get more brand-like results.
if (names.size() < max) {
String topic = deriveTopic(request);
String regionStr = deriveRegionStr(request);
String q = topic + " " + regionStr + " официальный сайт";
try {
SerperSearchResult extra = serperSearchService.search(q);
names.addAll(extractCompanyNamesFromSerper(extra, max - names.size()));
} catch (Exception ignored) {
}
}
List<String> out = new ArrayList<>(names);
return out.size() > max ? out.subList(0, max) : out;
}
private String deriveTopic(MarketingAnalysisRequest request) {
String product = request != null ? request.getProduct() : null;
String niche = request != null ? request.getBusinessNiche() : null;
// For competitor/brand lookup, product is usually more specific than niche.
String topic = (product != null && !product.isBlank()) ? product : (niche != null ? niche : null);
return (topic == null || topic.isBlank()) ? "ниша" : topic.trim();
}
private String deriveRegionStr(MarketingAnalysisRequest request) {
List<String> regions = request != null ? request.getRegion() : null;
return (regions != null && !regions.isEmpty()) ? String.join(", ", regions) : "Казахстан";
}
private List<String> extractCompanyNamesFromSerper(SerperSearchResult result, int limit) {
int max = Math.max(0, Math.min(limit, 20));
if (max == 0 || result == null || result.items() == null) {
return List.of();
}
LinkedHashSet<String> out = new LinkedHashSet<>();
for (SerperSearchItem item : result.items()) {
if (item == null) {
continue;
}
String name = extractCompanyNameCandidate(item.title());
if (isGoodCompanyNameCandidate(name)) {
out.add(name);
} else {
String fromHost = extractCompanyNameFromHost(item.sourceHost());
if (isGoodCompanyNameCandidate(fromHost)) {
out.add(fromHost);
}
}
if (out.size() >= max) {
break;
}
}
return new ArrayList<>(out);
}
private String extractCompanyNameFromHost(String host) {
if (host == null) {
return null;
}
String h = host.trim().toLowerCase(Locale.ROOT);
if (h.isEmpty()) {
return null;
}
h = h.startsWith("www.") ? h.substring(4) : h;
// Ignore obvious aggregators/social networks.
String[] blocked = new String[] {
"google.", "yandex.", "2gis.", "instagram.", "facebook.", "vk.", "tiktok.", "youtube.", "wikipedia."
};
for (String b : blocked) {
if (h.startsWith(b) || h.contains("." + b) || h.equals(b)) {
return null;
}
}
String[] parts = h.split("\\.");
if (parts.length < 2) {
return null;
}
String sld = parts[0];
if (sld.isBlank() || sld.length() > 40) {
return null;
}
// Basic "humanization" of domain label.
String cleaned = sld.replaceAll("[^a-z0-9\\-]+", "").replace('-', ' ').trim();
if (cleaned.isEmpty()) {
return null;
}
// Title-case first character
return cleaned.substring(0, 1).toUpperCase(Locale.ROOT) + cleaned.substring(1);
}
private String extractCompanyNameCandidate(String title) {
if (title == null) {
return null;
}
String t = title.replace('\u00A0', ' ').trim();
if (t.isEmpty()) {
return null;
}
// Keep the left-most "brand-like" chunk: "Brand — ...", "Brand | ...", "Brand -
// ..."
String[] seps = new String[] { " — ", " - ", " | ", " – ", ":", "|", "—", "–" };
for (String sep : seps) {
int idx = t.indexOf(sep);
if (idx > 1) {
t = t.substring(0, idx).trim();
break;
}
}
t = t.replaceAll("[\"“”«»]+", "").trim();
if (t.isEmpty()) {
return null;
}
// Trim common location tails: "... Алматы", "... Астана", "... Казахстан"
t = t.replaceAll("\\s+(Алматы|Астана|Казахстан)\\s*$", "").trim();
return t.isEmpty() ? null : t;
}
private boolean isGoodCompanyNameCandidate(String s) {
if (s == null) {
return false;
}
String t = s.trim();
if (t.length() < 2 || t.length() > 60) {
return false;
}
String lower = t.toLowerCase(Locale.ROOT);
// Exclude obvious non-brands (directories/articles)
String[] bad = new String[] {
"топ", "рейтинг", "обзор", "лучшие", "лучших", "каталог", "список", "цены", "стоимость",
"отзывы", "адрес", "телефон", "контакты", "как выбрать", "вакансии", "купить", "доставка"
};
for (String b : bad) {
if (lower.contains(b)) {
return false;
}
}
// Too many words is usually an article title, not a brand
if (t.split("\\s+").length > 5) {
return false;
}
return true;
}
private void replacePlaceholderCompanyKeysIfNeeded(MarketingAnalysisResponseV2 parsed,
List<String> competitorCompanies) {
if (parsed == null || competitorCompanies == null || competitorCompanies.isEmpty()) {
return;
}
if (parsed.getSections() == null || parsed.getSections().getCompetitiveEnvironment() == null) {
return;
}
MarketingAnalysisResponseV2.CompetitiveEnvironment ce = parsed.getSections().getCompetitiveEnvironment();
Map<String, Integer> dvb = ce.getDigitalVisibilityBenchmarks();
if (dvb != null && !dvb.isEmpty()) {
boolean hasPlaceholders = dvb.keySet().stream().anyMatch(this::isPlaceholderCompetitorKey);
if (hasPlaceholders) {
List<Integer> values = new ArrayList<>(dvb.values());
LinkedHashMap<String, Integer> rewritten = new LinkedHashMap<>();
int n = Math.min(values.size(), competitorCompanies.size());
for (int i = 0; i < n; i++) {
rewritten.put(competitorCompanies.get(i), values.get(i));
}
ce.setDigitalVisibilityBenchmarks(rewritten);
}
}
Map<String, List<String>> strengths = ce.getCompetitorStrengths();
if (strengths != null && !strengths.isEmpty()) {
boolean hasPlaceholders = strengths.keySet().stream().anyMatch(this::isPlaceholderCompetitorKey);
if (hasPlaceholders) {
List<List<String>> values = new ArrayList<>(strengths.values());
LinkedHashMap<String, List<String>> rewritten = new LinkedHashMap<>();
int n = Math.min(values.size(), competitorCompanies.size());
for (int i = 0; i < n; i++) {
rewritten.put(competitorCompanies.get(i), values.get(i));
}
ce.setCompetitorStrengths(rewritten);
}
}
}
/**
@@ -855,7 +1130,8 @@ public class MarketingAnalysisService {
* @param request Запрос на анализ для динамического формирования примера
* age_structure
*/
private MarketingAnalysisResponseV2 buildAnalysisV2ExampleForPrompt(MarketingAnalysisRequest request) {
private MarketingAnalysisResponseV2 buildAnalysisV2ExampleForPrompt(MarketingAnalysisRequest request,
List<String> competitorCompanies) {
MarketingAnalysisResponseV2 response = new MarketingAnalysisResponseV2();
response.setReportTitle("EXAMPLE__REPORT_TITLE");
@@ -946,17 +1222,19 @@ public class MarketingAnalysisService {
ceMetrics.setBenchmarkAstana("EXAMPLE__BENCHMARK_ASTANA");
competitiveEnvironment.setMetrics(ceMetrics);
Map<String, Integer> digitalVisibilityBenchmarks = new LinkedHashMap<>();
digitalVisibilityBenchmarks.put("competitor_a", -9999);
digitalVisibilityBenchmarks.put("your_business", -9999);
digitalVisibilityBenchmarks.put("lead", -9999);
digitalVisibilityBenchmarks.put("payment", -9999);
digitalVisibilityBenchmarks.put("others", -9999);
List<String> companies = (competitorCompanies != null) ? competitorCompanies : List.of();
if (companies.isEmpty()) {
companies = List.of("Компания 1", "Компания 2", "Компания 3", "Компания 4", "Компания 5");
}
for (int i = 0; i < Math.min(companies.size(), 5); i++) {
digitalVisibilityBenchmarks.put(companies.get(i), -9999);
}
competitiveEnvironment.setDigitalVisibilityBenchmarks(digitalVisibilityBenchmarks);
Map<String, List<String>> competitorStrengths = new LinkedHashMap<>();
competitorStrengths.put("your_business", List.of("EXAMPLE__STRENGTH_1", "EXAMPLE__STRENGTH_2"));
competitorStrengths.put("competitor_a", List.of("EXAMPLE__COMP_A_STRENGTH_1", "EXAMPLE__COMP_A_STRENGTH_2"));
competitorStrengths.put("competitor_b", List.of("EXAMPLE__COMP_B_STRENGTH_1", "EXAMPLE__COMP_B_STRENGTH_2"));
competitorStrengths.put("competitor_c", List.of("EXAMPLE__COMP_C_STRENGTH_1", "EXAMPLE__COMP_C_STRENGTH_2"));
for (int i = 0; i < Math.min(companies.size(), 5); i++) {
competitorStrengths.put(companies.get(i),
List.of("EXAMPLE__COMPETITOR_STRENGTH_1", "EXAMPLE__COMPETITOR_STRENGTH_2"));
}
competitiveEnvironment.setCompetitorStrengths(competitorStrengths);
competitiveEnvironment.setConclusion("EXAMPLE__COMPETITIVE_ENVIRONMENT_CONCLUSION");
sections.setCompetitiveEnvironment(competitiveEnvironment);