renolation/cdp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

runable

Browse Source
This commit is contained in:
renolation
2026-05-25 10:16:31 +07:00
parent a428170fef
commit c5e980aa52
21 changed files with 6172 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
ingestion/ingest/internal/dedup/dedup.go
View File

@@ -39,12 +39,14 @@ func (d *redisDedup) CheckAndSet(ctx context.Context, workspaceID, messageID str
Ex(d.ttl).
Build()
resp := d.client.Do(ctx, cmd)
if err := resp.Error(); err != nil {
return false, fmt.Errorf("dedup setnx: %w", err)
}
// SET with NX returns "OK" when set, nil reply when key already exists.
if resp.IsNil() {
err := resp.Error()
// SET NX returns nil reply when the key already exists; rueidis surfaces
// that as a "redis nil" error, which is *not* a real failure.
if rueidis.IsRedisNil(err) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("dedup setnx: %w", err)
}
return true, nil
}

11
ingestion/ingest/internal/kafka/producer.go
View File

@@ -33,7 +33,8 @@ func NewProducer(brokers []string, topicIngest, topicDLQ, topicRetry string, log
kgo.ProducerLinger(5_000_000), // 5ms linger -> batch small bursts
kgo.ProducerBatchCompression(kgo.ZstdCompression()),
kgo.MaxBufferedRecords(100_000),
kgo.RequiredAcks(kgo.LeaderAck()),
// franz-go enables idempotent writes by default, which requires acks=all.
kgo.RequiredAcks(kgo.AllISRAcks()),
kgo.ClientID("cdp-ingest"),
)
if err != nil {
@@ -57,6 +58,12 @@ func (p *Producer) Close() {
}
// Produce sends an event to the happy-path topic. Fire-and-forget.
//
// We detach the request's cancellation from the produce call: the HTTP
// handler returns 200 as soon as the record is buffered, after which the
// request context is cancelled. franz-go honours that cancellation and
// drops the buffered record. context.WithoutCancel preserves values for
// tracing but removes the deadline / Done signal.
func (p *Producer) Produce(ctx context.Context, ev *model.IngestedEvent) error {
payload, err := json.Marshal(ev)
if err != nil {
@@ -72,7 +79,7 @@ func (p *Producer) Produce(ctx context.Context, ev *model.IngestedEvent) error {
{Key: "type", Value: []byte(ev.Type)},
},
}
p.client.Produce(ctx, rec, func(r *kgo.Record, err error) {
p.client.Produce(context.WithoutCancel(ctx), rec, func(r *kgo.Record, err error) {
if err != nil {
p.log.Error("kafka produce failed",
zap.String("topic", r.Topic),