From 322a08f3dce52a2ecbe4e6f6fcd3653ff403fc84 Mon Sep 17 00:00:00 2001 From: balex Date: Fri, 27 Feb 2026 04:41:40 +0100 Subject: [PATCH] kafka --- .gitlab-ci.yml | 10 ++-- .../controller/AnalyticsController.java | 7 +-- .../analytics/dto/DailyStatsResponse.java | 4 +- .../analytics/dto/DashboardResponse.java | 4 +- .../com/posthub/analytics/dto/UserEvent.java | 4 +- .../posthub/analytics/model/DailyStats.java | 13 +---- .../com/posthub/analytics/model/EventLog.java | 12 ++--- .../posthub/analytics/model/UserStats.java | 10 +--- .../service/AnalyticsQueryService.java | 12 +---- .../analytics/service/AnalyticsService.java | 18 ++----- .../migration/V1__create_analytics_tables.sql | 47 ++++++++----------- .../com/balex/rag/model/dto/UserEvent.java | 3 +- 12 files changed, 42 insertions(+), 102 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12b9372..8fe8ad1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,7 +162,7 @@ deploy-rag: export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} docker compose -f docker-compose.yml -f docker-compose.prod.yml pull rag-service docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d rag-service - docker image prune -f + docker image prune -af ENDSSH deploy-gateway: @@ -177,7 +177,7 @@ deploy-gateway: export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} docker compose -f docker-compose.yml -f docker-compose.prod.yml pull gateway-service docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d gateway-service - docker image prune -f + docker image prune -af ENDSSH deploy-analytics: @@ -192,7 +192,7 @@ deploy-analytics: export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} docker compose -f docker-compose.yml -f docker-compose.prod.yml pull analytics-service docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d analytics-service - docker image prune -f + docker image prune -af ENDSSH # Deploy all services at once (manual trigger) @@ -209,5 +209,5 @@ deploy-all: export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} docker compose -f docker-compose.yml -f docker-compose.prod.yml pull docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d - docker image prune -f - ENDSSH + docker image prune -af + ENDSSH \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/controller/AnalyticsController.java b/analytics-service/src/main/java/com/posthub/analytics/controller/AnalyticsController.java index 6d39d55..989f993 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/controller/AnalyticsController.java +++ b/analytics-service/src/main/java/com/posthub/analytics/controller/AnalyticsController.java @@ -34,9 +34,4 @@ public class AnalyticsController { return ResponseEntity.ok(queryService.getDailyStats(days)); } - @GetMapping("/tokens/usage") - public ResponseEntity> getTokenUsage( - @RequestParam(defaultValue = "30") int days) { - return ResponseEntity.ok(queryService.getTokenUsage(days)); - } -} +} \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/dto/DailyStatsResponse.java b/analytics-service/src/main/java/com/posthub/analytics/dto/DailyStatsResponse.java index ca7b6e0..fe2a82b 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/dto/DailyStatsResponse.java +++ b/analytics-service/src/main/java/com/posthub/analytics/dto/DailyStatsResponse.java @@ -15,9 +15,7 @@ public class DailyStatsResponse { private LocalDate date; private Long queries; - private Long tokensUsed; - private Long ragHits; private Long newUsers; private Long newChats; private Integer activeUsers; -} +} \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/dto/DashboardResponse.java b/analytics-service/src/main/java/com/posthub/analytics/dto/DashboardResponse.java index f448dc3..6d324ce 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/dto/DashboardResponse.java +++ b/analytics-service/src/main/java/com/posthub/analytics/dto/DashboardResponse.java @@ -15,9 +15,7 @@ import java.util.Map; public class DashboardResponse { private Long totalQueries; - private Long totalTokensUsed; - private Long totalRagHits; private Long totalUsers; private Integer activeUsersToday; private Map eventBreakdown; -} +} \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/dto/UserEvent.java b/analytics-service/src/main/java/com/posthub/analytics/dto/UserEvent.java index d2cab04..4264a40 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/dto/UserEvent.java +++ b/analytics-service/src/main/java/com/posthub/analytics/dto/UserEvent.java @@ -18,8 +18,6 @@ public class UserEvent { private EventType type; private String userId; private String chatId; - private Integer tokensUsed; - private Integer documentsFound; private Instant timestamp; public enum EventType { @@ -28,4 +26,4 @@ public class UserEvent { CHAT_DELETED, QUERY_SENT } -} +} \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/model/DailyStats.java b/analytics-service/src/main/java/com/posthub/analytics/model/DailyStats.java index ac4afdc..914f3ca 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/model/DailyStats.java +++ b/analytics-service/src/main/java/com/posthub/analytics/model/DailyStats.java @@ -7,7 +7,7 @@ import java.time.LocalDate; @Entity @Table(name = "daily_stats", uniqueConstraints = { - @UniqueConstraint(columnNames = "statsDate") + @UniqueConstraint(columnNames = "stats_date") }) @Getter @Setter @@ -26,15 +26,6 @@ public class DailyStats { @Builder.Default private Long totalQueries = 0L; - @Builder.Default - private Long totalTokensUsed = 0L; - - @Builder.Default - private Long totalRagHits = 0L; - - @Builder.Default - private Long totalDocumentsFound = 0L; - @Builder.Default private Long newUsers = 0L; @@ -46,4 +37,4 @@ public class DailyStats { @Builder.Default private Integer activeUsers = 0; -} +} \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/model/EventLog.java b/analytics-service/src/main/java/com/posthub/analytics/model/EventLog.java index 60bbd7f..74e1c43 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/model/EventLog.java +++ b/analytics-service/src/main/java/com/posthub/analytics/model/EventLog.java @@ -8,9 +8,9 @@ import java.time.LocalDate; @Entity @Table(name = "event_log", indexes = { - @Index(name = "idx_event_log_type", columnList = "eventType"), - @Index(name = "idx_event_log_user", columnList = "userId"), - @Index(name = "idx_event_log_date", columnList = "eventDate") + @Index(name = "idx_event_log_type", columnList = "event_type"), + @Index(name = "idx_event_log_user", columnList = "user_id"), + @Index(name = "idx_event_log_date", columnList = "event_date") }) @Getter @Setter @@ -31,13 +31,9 @@ public class EventLog { private String chatId; - private Integer tokensUsed; - - private Integer documentsFound; - @Column(nullable = false) private Instant eventTimestamp; @Column(nullable = false) private LocalDate eventDate; -} +} \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/model/UserStats.java b/analytics-service/src/main/java/com/posthub/analytics/model/UserStats.java index 4eac0cd..0548578 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/model/UserStats.java +++ b/analytics-service/src/main/java/com/posthub/analytics/model/UserStats.java @@ -7,7 +7,7 @@ import java.time.Instant; @Entity @Table(name = "user_stats", uniqueConstraints = { - @UniqueConstraint(columnNames = "userId") + @UniqueConstraint(columnNames = "user_id") }) @Getter @Setter @@ -26,16 +26,10 @@ public class UserStats { @Builder.Default private Long totalQueries = 0L; - @Builder.Default - private Long totalTokensUsed = 0L; - - @Builder.Default - private Long totalRagHits = 0L; - @Builder.Default private Integer totalChats = 0; private Instant firstSeen; private Instant lastActive; -} +} \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsQueryService.java b/analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsQueryService.java index f9ebc16..7eac9cf 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsQueryService.java +++ b/analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsQueryService.java @@ -35,8 +35,6 @@ public class AnalyticsQueryService { .findByStatsDateBetweenOrderByStatsDateAsc(thirtyDaysAgo, today); long totalQueries = last30Days.stream().mapToLong(DailyStats::getTotalQueries).sum(); - long totalTokens = last30Days.stream().mapToLong(DailyStats::getTotalTokensUsed).sum(); - long totalRagHits = last30Days.stream().mapToLong(DailyStats::getTotalRagHits).sum(); long totalUsers = userStatsRepository.count(); Integer activeToday = eventLogRepository.countActiveUsers(today, today); @@ -48,8 +46,6 @@ public class AnalyticsQueryService { return DashboardResponse.builder() .totalQueries(totalQueries) - .totalTokensUsed(totalTokens) - .totalRagHits(totalRagHits) .totalUsers(totalUsers) .activeUsersToday(activeToday != null ? activeToday : 0) .eventBreakdown(breakdown) @@ -71,19 +67,13 @@ public class AnalyticsQueryService { return userStatsRepository.findActiveUsersSince(since); } - public List getTokenUsage(int days) { - return getDailyStats(days); - } - private DailyStatsResponse toResponse(DailyStats ds) { return DailyStatsResponse.builder() .date(ds.getStatsDate()) .queries(ds.getTotalQueries()) - .tokensUsed(ds.getTotalTokensUsed()) - .ragHits(ds.getTotalRagHits()) .newUsers(ds.getNewUsers()) .newChats(ds.getNewChats()) .activeUsers(ds.getActiveUsers()) .build(); } -} +} \ No newline at end of file diff --git a/analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsService.java b/analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsService.java index d3f5bda..e4166fc 100644 --- a/analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsService.java +++ b/analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsService.java @@ -35,8 +35,6 @@ public class AnalyticsService { .eventType(event.getType().name()) .userId(event.getUserId()) .chatId(event.getChatId()) - .tokensUsed(event.getTokensUsed()) - .documentsFound(event.getDocumentsFound()) .eventTimestamp(timestamp) .eventDate(eventDate) .build(); @@ -59,12 +57,7 @@ public class AnalyticsService { case USER_CREATED -> daily.setNewUsers(daily.getNewUsers() + 1); case CHAT_CREATED -> daily.setNewChats(daily.getNewChats() + 1); case CHAT_DELETED -> daily.setDeletedChats(daily.getDeletedChats() + 1); - case QUERY_SENT -> { - daily.setTotalQueries(daily.getTotalQueries() + 1); - if (event.getTokensUsed() != null) { - daily.setTotalTokensUsed(daily.getTotalTokensUsed() + event.getTokensUsed()); - } - } + case QUERY_SENT -> daily.setTotalQueries(daily.getTotalQueries() + 1); } dailyStatsRepository.save(daily); @@ -81,15 +74,10 @@ public class AnalyticsService { switch (event.getType()) { case CHAT_CREATED -> userStats.setTotalChats(userStats.getTotalChats() + 1); - case QUERY_SENT -> { - userStats.setTotalQueries(userStats.getTotalQueries() + 1); - if (event.getTokensUsed() != null) { - userStats.setTotalTokensUsed(userStats.getTotalTokensUsed() + event.getTokensUsed()); - } - } + case QUERY_SENT -> userStats.setTotalQueries(userStats.getTotalQueries() + 1); default -> { /* no user-level aggregation needed */ } } userStatsRepository.save(userStats); } -} +} \ No newline at end of file diff --git a/analytics-service/src/main/resources/db/migration/V1__create_analytics_tables.sql b/analytics-service/src/main/resources/db/migration/V1__create_analytics_tables.sql index fabd492..2ea17d4 100644 --- a/analytics-service/src/main/resources/db/migration/V1__create_analytics_tables.sql +++ b/analytics-service/src/main/resources/db/migration/V1__create_analytics_tables.sql @@ -1,12 +1,10 @@ CREATE TABLE event_log ( - id BIGSERIAL PRIMARY KEY, - event_type VARCHAR(50) NOT NULL, - user_id VARCHAR(255) NOT NULL, - chat_id VARCHAR(255), - tokens_used INTEGER, - documents_found INTEGER, - event_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - event_date DATE NOT NULL + id BIGSERIAL PRIMARY KEY, + event_type VARCHAR(50) NOT NULL, + user_id VARCHAR(255) NOT NULL, + chat_id VARCHAR(255), + event_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + event_date DATE NOT NULL ); CREATE INDEX idx_event_log_type ON event_log(event_type); @@ -14,25 +12,20 @@ CREATE INDEX idx_event_log_user ON event_log(user_id); CREATE INDEX idx_event_log_date ON event_log(event_date); CREATE TABLE daily_stats ( - id BIGSERIAL PRIMARY KEY, - stats_date DATE NOT NULL UNIQUE, - total_queries BIGINT DEFAULT 0, - total_tokens_used BIGINT DEFAULT 0, - total_rag_hits BIGINT DEFAULT 0, - total_documents_found BIGINT DEFAULT 0, - new_users BIGINT DEFAULT 0, - new_chats BIGINT DEFAULT 0, - deleted_chats BIGINT DEFAULT 0, - active_users INTEGER DEFAULT 0 + id BIGSERIAL PRIMARY KEY, + stats_date DATE NOT NULL UNIQUE, + total_queries BIGINT DEFAULT 0, + new_users BIGINT DEFAULT 0, + new_chats BIGINT DEFAULT 0, + deleted_chats BIGINT DEFAULT 0, + active_users INTEGER DEFAULT 0 ); CREATE TABLE user_stats ( - id BIGSERIAL PRIMARY KEY, - user_id VARCHAR(255) NOT NULL UNIQUE, - total_queries BIGINT DEFAULT 0, - total_tokens_used BIGINT DEFAULT 0, - total_rag_hits BIGINT DEFAULT 0, - total_chats INTEGER DEFAULT 0, - first_seen TIMESTAMP WITH TIME ZONE, - last_active TIMESTAMP WITH TIME ZONE -); + id BIGSERIAL PRIMARY KEY, + user_id VARCHAR(255) NOT NULL UNIQUE, + total_queries BIGINT DEFAULT 0, + total_chats INTEGER DEFAULT 0, + first_seen TIMESTAMP WITH TIME ZONE, + last_active TIMESTAMP WITH TIME ZONE +); \ No newline at end of file diff --git a/rag-service/src/main/java/com/balex/rag/model/dto/UserEvent.java b/rag-service/src/main/java/com/balex/rag/model/dto/UserEvent.java index 93f1101..43f716c 100644 --- a/rag-service/src/main/java/com/balex/rag/model/dto/UserEvent.java +++ b/rag-service/src/main/java/com/balex/rag/model/dto/UserEvent.java @@ -30,8 +30,7 @@ public class UserEvent { USER_CREATED, CHAT_CREATED, CHAT_DELETED, - QUERY_SENT, - RAG_CONTEXT_HIT + QUERY_SENT } }