init ingestion

2026-05-24 22:59:24 +07:00
commit 4e8c11d545
80 changed files with 5639 additions and 0 deletions
--- a/ingestion/ingest/internal/model/event.go
+++ b/ingestion/ingest/internal/model/event.go
@@ -0,0 +1,81 @@
+// Package model holds the wire and domain types passed between layers.
+package model
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// EventType is the Segment-compatible call kind.
+type EventType string
+
+const (
+	EventTypeTrack    EventType = "track"
+	EventTypeIdentify EventType = "identify"
+	EventTypePage     EventType = "page"
+	EventTypeGroup    EventType = "group"
+	EventTypeAlias    EventType = "alias"
+	EventTypeScreen   EventType = "screen"
+)
+
+// RawEvent is the parsed-but-not-yet-validated payload from a client.
+// We keep Properties / Traits / Context as json.RawMessage so the handler can
+// pass them through to the service untouched; flattening happens in service.
+type RawEvent struct {
+	Type        EventType       `json:"type"        validate:"required,oneof=track identify page group alias screen"`
+	MessageID   string          `json:"messageId"   validate:"required,max=128"`
+	AnonymousID string          `json:"anonymousId" validate:"max=128"`
+	UserID      string          `json:"userId"      validate:"max=128"`
+	GroupID     string          `json:"groupId"     validate:"max=128"`
+	Event       string          `json:"event"       validate:"max=255"`
+	Name        string          `json:"name"        validate:"max=255"`
+	Category    string          `json:"category"    validate:"max=255"`
+	Properties  json.RawMessage `json:"properties"`
+	Traits      json.RawMessage `json:"traits"`
+	Context     json.RawMessage `json:"context"`
+	Timestamp   *time.Time      `json:"timestamp"`
+	SentAt      *time.Time      `json:"sentAt"`
+}
+
+// BatchEnvelope is the body of /batch — Segment-compatible.
+type BatchEnvelope struct {
+	Batch    []RawEvent      `json:"batch" validate:"required,min=1,max=1000,dive"`
+	SentAt   *time.Time      `json:"sentAt"`
+	Context  json.RawMessage `json:"context"`
+}
+
+// IngestedEvent is the canonical record we push onto Kafka. Flat fields,
+// timestamps already normalized, payload sanitized.
+type IngestedEvent struct {
+	WorkspaceID string                 `json:"workspace_id"`
+	SourceID    string                 `json:"source_id"`
+	MessageID   string                 `json:"message_id"`
+	Type        EventType              `json:"type"`
+	AnonymousID string                 `json:"anonymous_id,omitempty"`
+	UserID      string                 `json:"user_id,omitempty"`
+	GroupID     string                 `json:"group_id,omitempty"`
+	Event       string                 `json:"event,omitempty"`
+	Name        string                 `json:"name,omitempty"`
+	Category    string                 `json:"category,omitempty"`
+	Properties  map[string]any         `json:"properties,omitempty"`
+	Traits      map[string]any         `json:"traits,omitempty"`
+	Context     map[string]any         `json:"context,omitempty"`
+	IP          string                 `json:"ip,omitempty"`
+	UserAgent   string                 `json:"user_agent,omitempty"`
+
+	Timestamp  time.Time `json:"timestamp"`
+	SentAt     time.Time `json:"sent_at"`
+	ReceivedAt time.Time `json:"received_at"`
+}
+
+// PartitionKey returns the key used for Kafka partitioning. We use
+// anonymous_id to keep identity-stitching ordering per visitor.
+func (e *IngestedEvent) PartitionKey() string {
+	if e.AnonymousID != "" {
+		return e.AnonymousID
+	}
+	if e.UserID != "" {
+		return e.UserID
+	}
+	return e.MessageID
+}