renolation/cdp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

runable

Browse Source
This commit is contained in:
renolation
2026-05-25 10:16:31 +07:00
parent a428170fef
commit c5e980aa52
21 changed files with 6172 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
ingestion/Makefile
View File

@@ -9,12 +9,54 @@
# Configuration
# ---------------------------------------------------------------------------
POSTGRES_DSN ?= postgres://cdp:cdp@localhost:5432/cdp?sslmode=disable
CLICKHOUSE_DSN ?= clickhouse://default:@localhost:9000/cdp
MIGRATE_BIN ?= migrate
# Auto-load .env if present, then export every variable so child processes
# (go run, docker run, the apply script) inherit them. GNU Make 3.81 (macOS
# default) ignores bare `export`, so we list the keys explicitly below after
# all variables have been declared.
ifneq (,$(wildcard ./.env))
include .env
endif
POSTGRES_DSN ?= postgres://renolation:renolation@103.188.82.191:5432/ingestion?sslmode=disable
# For dockerized `migrate`: same DSN works because the target is reachable from
# the container too (it's a public host, not host.docker.internal).
POSTGRES_DSN_DOCKER ?= $(POSTGRES_DSN)
CLICKHOUSE_ADDR ?= 192.168.1.60:8123
CLICKHOUSE_DB ?= analytics
CLICKHOUSE_USER ?= renolation
CLICKHOUSE_PASSWORD ?= renolation
CLICKHOUSE_SECURE ?= 0
MIGRATE_IMAGE ?= migrate/migrate:v4.17.1
MIGRATIONS_DIR := infra/migrations
CLICKHOUSE_DIR := infra/clickhouse
# Re-export everything that the Go services and the apply script read.
# Listed explicitly so this works on GNU Make 3.81 (macOS default).
export POSTGRES_DSN REDIS_ADDR KAFKA_BROKERS
export CLICKHOUSE_ADDR CLICKHOUSE_DB CLICKHOUSE_USER CLICKHOUSE_PASSWORD CLICKHOUSE_SECURE
export INGEST_HTTP_ADDR INGEST_LOG_LEVEL INGEST_PAYLOAD_LIMIT_KB INGEST_BATCH_LIMIT_KB
export INGEST_LATE_EVENT_HOURS INGEST_DEDUP_TTL_HOURS INGEST_WRITE_KEY_CACHE_TTL_SECONDS
export INGEST_LOG_PAYLOAD_ON_SUCCESS INGEST_LOG_PAYLOAD_ON_ERROR INGEST_SHUTDOWN_TIMEOUT_SECONDS
export KAFKA_TOPIC_INGEST KAFKA_TOPIC_DLQ KAFKA_TOPIC_RETRY
export BULKER_HTTP_ADDR BULKER_LOG_LEVEL BULKER_KAFKA_GROUP BULKER_BATCH_SIZE
export BULKER_BATCH_INTERVAL_SECONDS BULKER_SHUTDOWN_TIMEOUT_SECONDS
export ROTOR_PORT ROTOR_LOG_LEVEL ROTOR_ISOLATE_MEMORY_MB ROTOR_FUNCTION_TIMEOUT_MS
# `migrate` CLI: prefer the locally-installed binary if it exists, otherwise
# run the official Docker image. Set MIGRATE_BIN to override.
MIGRATE_BIN ?= $(shell command -v migrate 2>/dev/null)
ifeq ($(MIGRATE_BIN),)
MIGRATE = docker run --rm \
-v $(CURDIR)/$(MIGRATIONS_DIR):/migrations \
$(MIGRATE_IMAGE) -path=/migrations -database "$(POSTGRES_DSN_DOCKER)"
MIGRATE_CREATE = docker run --rm \
-v $(CURDIR)/$(MIGRATIONS_DIR):/migrations \
$(MIGRATE_IMAGE) create -ext sql -dir /migrations -seq
else
MIGRATE = $(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)"
MIGRATE_CREATE = $(MIGRATE_BIN) create -ext sql -dir $(MIGRATIONS_DIR) -seq
endif
# ---------------------------------------------------------------------------
# Help
# ---------------------------------------------------------------------------
@@ -61,16 +103,16 @@ logs:
migrate/new:
@if [ -z "$(name)" ]; then echo "usage: make migrate/new name=add_xxx"; exit 1; fi
$(MIGRATE_BIN) create -ext sql -dir $(MIGRATIONS_DIR) -seq $(name)
$(MIGRATE_CREATE) $(name)
migrate/up:
$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" up
$(MIGRATE) up
migrate/down:
$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" down 1
$(MIGRATE) down 1
migrate/status:
$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" version
$(MIGRATE) version
# ---------------------------------------------------------------------------
# ClickHouse DDL
@@ -90,12 +132,22 @@ run/ingest:
cd ingest && go run ./cmd/server
run/bulker:
@echo ">>> CLICKHOUSE_ADDR=$$CLICKHOUSE_ADDR CLICKHOUSE_SECURE=$$CLICKHOUSE_SECURE"
cd bulker && go run ./cmd/server
debug/env:
@echo "CLICKHOUSE_ADDR=$$CLICKHOUSE_ADDR"
@echo "CLICKHOUSE_SECURE=$$CLICKHOUSE_SECURE"
@echo "POSTGRES_DSN=$$POSTGRES_DSN"
@echo "-- child env --"
@env | grep -E '^(CLICKHOUSE|POSTGRES|KAFKA|INGEST|BULKER)' | sort
run/rotor:
@cd rotor && [ -d node_modules ] || npm install
cd rotor && npm run dev
run/console:
@cd console && [ -d node_modules ] || npm install
cd console && npm run dev
# ---------------------------------------------------------------------------