balex74/post-hub-platform-k7m2xq9wnp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kafka

Browse Source
This commit is contained in:
balex
2026-02-27 04:41:40 +01:00
parent 75a4c6a99e
commit 322a08f3dc
12 changed files with 42 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.gitlab-ci.yml
View File

@@ -162,7 +162,7 @@ deploy-rag:
export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
docker compose -f docker-compose.yml -f docker-compose.prod.yml pull rag-service docker compose -f docker-compose.yml -f docker-compose.prod.yml pull rag-service
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d rag-service docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d rag-service
docker image prune -f docker image prune -af
ENDSSH ENDSSH
deploy-gateway: deploy-gateway:
@@ -177,7 +177,7 @@ deploy-gateway:
export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
docker compose -f docker-compose.yml -f docker-compose.prod.yml pull gateway-service docker compose -f docker-compose.yml -f docker-compose.prod.yml pull gateway-service
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d gateway-service docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d gateway-service
docker image prune -f docker image prune -af
ENDSSH ENDSSH
deploy-analytics: deploy-analytics:
@@ -192,7 +192,7 @@ deploy-analytics:
export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
docker compose -f docker-compose.yml -f docker-compose.prod.yml pull analytics-service docker compose -f docker-compose.yml -f docker-compose.prod.yml pull analytics-service
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d analytics-service docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d analytics-service
docker image prune -f docker image prune -af
ENDSSH ENDSSH
# Deploy all services at once (manual trigger) # Deploy all services at once (manual trigger)
@@ -209,5 +209,5 @@ deploy-all:
export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
docker compose -f docker-compose.yml -f docker-compose.prod.yml pull docker compose -f docker-compose.yml -f docker-compose.prod.yml pull
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
docker image prune -f docker image prune -af
ENDSSH ENDSSH

7
analytics-service/src/main/java/com/posthub/analytics/controller/AnalyticsController.java
View File

@@ -34,9 +34,4 @@ public class AnalyticsController {
return ResponseEntity.ok(queryService.getDailyStats(days)); return ResponseEntity.ok(queryService.getDailyStats(days));
} }
@GetMapping("/tokens/usage") }
public ResponseEntity<List<DailyStatsResponse>> getTokenUsage(
@RequestParam(defaultValue = "30") int days) {
return ResponseEntity.ok(queryService.getTokenUsage(days));
}
}

4
analytics-service/src/main/java/com/posthub/analytics/dto/DailyStatsResponse.java
View File

@@ -15,9 +15,7 @@ public class DailyStatsResponse {
private LocalDate date; private LocalDate date;
private Long queries; private Long queries;
private Long tokensUsed;
private Long ragHits;
private Long newUsers; private Long newUsers;
private Long newChats; private Long newChats;
private Integer activeUsers; private Integer activeUsers;
} }

4
analytics-service/src/main/java/com/posthub/analytics/dto/DashboardResponse.java
View File

@@ -15,9 +15,7 @@ import java.util.Map;
public class DashboardResponse { public class DashboardResponse {
private Long totalQueries; private Long totalQueries;
private Long totalTokensUsed;
private Long totalRagHits;
private Long totalUsers; private Long totalUsers;
private Integer activeUsersToday; private Integer activeUsersToday;
private Map<String, Long> eventBreakdown; private Map<String, Long> eventBreakdown;
} }

4
analytics-service/src/main/java/com/posthub/analytics/dto/UserEvent.java
View File

@@ -18,8 +18,6 @@ public class UserEvent {
private EventType type; private EventType type;
private String userId; private String userId;
private String chatId; private String chatId;
private Integer tokensUsed;
private Integer documentsFound;
private Instant timestamp; private Instant timestamp;
public enum EventType { public enum EventType {
@@ -28,4 +26,4 @@ public class UserEvent {
CHAT_DELETED, CHAT_DELETED,
QUERY_SENT QUERY_SENT
} }
} }

13
analytics-service/src/main/java/com/posthub/analytics/model/DailyStats.java
View File

@@ -7,7 +7,7 @@ import java.time.LocalDate;
@Entity @Entity
@Table(name = "daily_stats", uniqueConstraints = { @Table(name = "daily_stats", uniqueConstraints = {
@UniqueConstraint(columnNames = "statsDate") @UniqueConstraint(columnNames = "stats_date")
}) })
@Getter @Getter
@Setter @Setter
@@ -26,15 +26,6 @@ public class DailyStats {
@Builder.Default @Builder.Default
private Long totalQueries = 0L; private Long totalQueries = 0L;
@Builder.Default
private Long totalTokensUsed = 0L;
@Builder.Default
private Long totalRagHits = 0L;
@Builder.Default
private Long totalDocumentsFound = 0L;
@Builder.Default @Builder.Default
private Long newUsers = 0L; private Long newUsers = 0L;
@@ -46,4 +37,4 @@ public class DailyStats {
@Builder.Default @Builder.Default
private Integer activeUsers = 0; private Integer activeUsers = 0;
} }

12
analytics-service/src/main/java/com/posthub/analytics/model/EventLog.java
View File

@@ -8,9 +8,9 @@ import java.time.LocalDate;
@Entity @Entity
@Table(name = "event_log", indexes = { @Table(name = "event_log", indexes = {
@Index(name = "idx_event_log_type", columnList = "eventType"), @Index(name = "idx_event_log_type", columnList = "event_type"),
@Index(name = "idx_event_log_user", columnList = "userId"), @Index(name = "idx_event_log_user", columnList = "user_id"),
@Index(name = "idx_event_log_date", columnList = "eventDate") @Index(name = "idx_event_log_date", columnList = "event_date")
}) })
@Getter @Getter
@Setter @Setter
@@ -31,13 +31,9 @@ public class EventLog {
private String chatId; private String chatId;
private Integer tokensUsed;
private Integer documentsFound;
@Column(nullable = false) @Column(nullable = false)
private Instant eventTimestamp; private Instant eventTimestamp;
@Column(nullable = false) @Column(nullable = false)
private LocalDate eventDate; private LocalDate eventDate;
} }

10
analytics-service/src/main/java/com/posthub/analytics/model/UserStats.java
View File

@@ -7,7 +7,7 @@ import java.time.Instant;
@Entity @Entity
@Table(name = "user_stats", uniqueConstraints = { @Table(name = "user_stats", uniqueConstraints = {
@UniqueConstraint(columnNames = "userId") @UniqueConstraint(columnNames = "user_id")
}) })
@Getter @Getter
@Setter @Setter
@@ -26,16 +26,10 @@ public class UserStats {
@Builder.Default @Builder.Default
private Long totalQueries = 0L; private Long totalQueries = 0L;
@Builder.Default
private Long totalTokensUsed = 0L;
@Builder.Default
private Long totalRagHits = 0L;
@Builder.Default @Builder.Default
private Integer totalChats = 0; private Integer totalChats = 0;
private Instant firstSeen; private Instant firstSeen;
private Instant lastActive; private Instant lastActive;
} }

12
analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsQueryService.java
View File

@@ -35,8 +35,6 @@ public class AnalyticsQueryService {
.findByStatsDateBetweenOrderByStatsDateAsc(thirtyDaysAgo, today); .findByStatsDateBetweenOrderByStatsDateAsc(thirtyDaysAgo, today);
long totalQueries = last30Days.stream().mapToLong(DailyStats::getTotalQueries).sum(); long totalQueries = last30Days.stream().mapToLong(DailyStats::getTotalQueries).sum();
long totalTokens = last30Days.stream().mapToLong(DailyStats::getTotalTokensUsed).sum();
long totalRagHits = last30Days.stream().mapToLong(DailyStats::getTotalRagHits).sum();
long totalUsers = userStatsRepository.count(); long totalUsers = userStatsRepository.count();
Integer activeToday = eventLogRepository.countActiveUsers(today, today); Integer activeToday = eventLogRepository.countActiveUsers(today, today);
@@ -48,8 +46,6 @@ public class AnalyticsQueryService {
return DashboardResponse.builder() return DashboardResponse.builder()
.totalQueries(totalQueries) .totalQueries(totalQueries)
.totalTokensUsed(totalTokens)
.totalRagHits(totalRagHits)
.totalUsers(totalUsers) .totalUsers(totalUsers)
.activeUsersToday(activeToday != null ? activeToday : 0) .activeUsersToday(activeToday != null ? activeToday : 0)
.eventBreakdown(breakdown) .eventBreakdown(breakdown)
@@ -71,19 +67,13 @@ public class AnalyticsQueryService {
return userStatsRepository.findActiveUsersSince(since); return userStatsRepository.findActiveUsersSince(since);
} }
public List<DailyStatsResponse> getTokenUsage(int days) {
return getDailyStats(days);
}
private DailyStatsResponse toResponse(DailyStats ds) { private DailyStatsResponse toResponse(DailyStats ds) {
return DailyStatsResponse.builder() return DailyStatsResponse.builder()
.date(ds.getStatsDate()) .date(ds.getStatsDate())
.queries(ds.getTotalQueries()) .queries(ds.getTotalQueries())
.tokensUsed(ds.getTotalTokensUsed())
.ragHits(ds.getTotalRagHits())
.newUsers(ds.getNewUsers()) .newUsers(ds.getNewUsers())
.newChats(ds.getNewChats()) .newChats(ds.getNewChats())
.activeUsers(ds.getActiveUsers()) .activeUsers(ds.getActiveUsers())
.build(); .build();
} }
} }

18
analytics-service/src/main/java/com/posthub/analytics/service/AnalyticsService.java
View File

@@ -35,8 +35,6 @@ public class AnalyticsService {
.eventType(event.getType().name()) .eventType(event.getType().name())
.userId(event.getUserId()) .userId(event.getUserId())
.chatId(event.getChatId()) .chatId(event.getChatId())
.tokensUsed(event.getTokensUsed())
.documentsFound(event.getDocumentsFound())
.eventTimestamp(timestamp) .eventTimestamp(timestamp)
.eventDate(eventDate) .eventDate(eventDate)
.build(); .build();
@@ -59,12 +57,7 @@ public class AnalyticsService {
case USER_CREATED -> daily.setNewUsers(daily.getNewUsers() + 1); case USER_CREATED -> daily.setNewUsers(daily.getNewUsers() + 1);
case CHAT_CREATED -> daily.setNewChats(daily.getNewChats() + 1); case CHAT_CREATED -> daily.setNewChats(daily.getNewChats() + 1);
case CHAT_DELETED -> daily.setDeletedChats(daily.getDeletedChats() + 1); case CHAT_DELETED -> daily.setDeletedChats(daily.getDeletedChats() + 1);
case QUERY_SENT -> { case QUERY_SENT -> daily.setTotalQueries(daily.getTotalQueries() + 1);
daily.setTotalQueries(daily.getTotalQueries() + 1);
if (event.getTokensUsed() != null) {
daily.setTotalTokensUsed(daily.getTotalTokensUsed() + event.getTokensUsed());
}
}
} }
dailyStatsRepository.save(daily); dailyStatsRepository.save(daily);
@@ -81,15 +74,10 @@ public class AnalyticsService {
switch (event.getType()) { switch (event.getType()) {
case CHAT_CREATED -> userStats.setTotalChats(userStats.getTotalChats() + 1); case CHAT_CREATED -> userStats.setTotalChats(userStats.getTotalChats() + 1);
case QUERY_SENT -> { case QUERY_SENT -> userStats.setTotalQueries(userStats.getTotalQueries() + 1);
userStats.setTotalQueries(userStats.getTotalQueries() + 1);
if (event.getTokensUsed() != null) {
userStats.setTotalTokensUsed(userStats.getTotalTokensUsed() + event.getTokensUsed());
}
}
default -> { /* no user-level aggregation needed */ } default -> { /* no user-level aggregation needed */ }
} }
userStatsRepository.save(userStats); userStatsRepository.save(userStats);
} }
} }

47
analytics-service/src/main/resources/db/migration/V1__create_analytics_tables.sql
View File

@@ -1,12 +1,10 @@
CREATE TABLE event_log ( CREATE TABLE event_log (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
event_type VARCHAR(50) NOT NULL, event_type VARCHAR(50) NOT NULL,
user_id VARCHAR(255) NOT NULL, user_id VARCHAR(255) NOT NULL,
chat_id VARCHAR(255), chat_id VARCHAR(255),
tokens_used INTEGER, event_timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
documents_found INTEGER, event_date DATE NOT NULL
event_timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
event_date DATE NOT NULL
); );
CREATE INDEX idx_event_log_type ON event_log(event_type); CREATE INDEX idx_event_log_type ON event_log(event_type);
@@ -14,25 +12,20 @@ CREATE INDEX idx_event_log_user ON event_log(user_id);
CREATE INDEX idx_event_log_date ON event_log(event_date); CREATE INDEX idx_event_log_date ON event_log(event_date);
CREATE TABLE daily_stats ( CREATE TABLE daily_stats (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
stats_date DATE NOT NULL UNIQUE, stats_date DATE NOT NULL UNIQUE,
total_queries BIGINT DEFAULT 0, total_queries BIGINT DEFAULT 0,
total_tokens_used BIGINT DEFAULT 0, new_users BIGINT DEFAULT 0,
total_rag_hits BIGINT DEFAULT 0, new_chats BIGINT DEFAULT 0,
total_documents_found BIGINT DEFAULT 0, deleted_chats BIGINT DEFAULT 0,
new_users BIGINT DEFAULT 0, active_users INTEGER DEFAULT 0
new_chats BIGINT DEFAULT 0,
deleted_chats BIGINT DEFAULT 0,
active_users INTEGER DEFAULT 0
); );
CREATE TABLE user_stats ( CREATE TABLE user_stats (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
user_id VARCHAR(255) NOT NULL UNIQUE, user_id VARCHAR(255) NOT NULL UNIQUE,
total_queries BIGINT DEFAULT 0, total_queries BIGINT DEFAULT 0,
total_tokens_used BIGINT DEFAULT 0, total_chats INTEGER DEFAULT 0,
total_rag_hits BIGINT DEFAULT 0, first_seen TIMESTAMP WITH TIME ZONE,
total_chats INTEGER DEFAULT 0, last_active TIMESTAMP WITH TIME ZONE
first_seen TIMESTAMP WITH TIME ZONE, );
last_active TIMESTAMP WITH TIME ZONE
);

3
rag-service/src/main/java/com/balex/rag/model/dto/UserEvent.java
View File

@@ -30,8 +30,7 @@ public class UserEvent {
USER_CREATED, USER_CREATED,
CHAT_CREATED, CHAT_CREATED,
CHAT_DELETED, CHAT_DELETED,
QUERY_SENT, QUERY_SENT
RAG_CONTEXT_HIT
} }
} }