.PHONY: help up down logs migrate/new migrate/up migrate/down migrate/status \ clickhouse/up clickhouse/down \ run/ingest run/bulker run/rotor run/console \ build/ingest build/bulker \ test test/integration \ lint fmt tidy # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- # Auto-load .env if present, then export every variable so child processes # (go run, docker run, the apply script) inherit them. GNU Make 3.81 (macOS # default) ignores bare `export`, so we list the keys explicitly below after # all variables have been declared. ifneq (,$(wildcard ./.env)) include .env endif POSTGRES_DSN ?= postgres://renolation:renolation@103.188.82.191:5432/ingestion?sslmode=disable # For dockerized `migrate`: same DSN works because the target is reachable from # the container too (it's a public host, not host.docker.internal). POSTGRES_DSN_DOCKER ?= $(POSTGRES_DSN) CLICKHOUSE_ADDR ?= 192.168.1.60:8123 CLICKHOUSE_DB ?= analytics CLICKHOUSE_USER ?= renolation CLICKHOUSE_PASSWORD ?= renolation CLICKHOUSE_SECURE ?= 0 MIGRATE_IMAGE ?= migrate/migrate:v4.17.1 MIGRATIONS_DIR := infra/migrations CLICKHOUSE_DIR := infra/clickhouse # Re-export everything that the Go services and the apply script read. # Listed explicitly so this works on GNU Make 3.81 (macOS default). export POSTGRES_DSN REDIS_ADDR KAFKA_BROKERS export CLICKHOUSE_ADDR CLICKHOUSE_DB CLICKHOUSE_USER CLICKHOUSE_PASSWORD CLICKHOUSE_SECURE export INGEST_HTTP_ADDR INGEST_LOG_LEVEL INGEST_PAYLOAD_LIMIT_KB INGEST_BATCH_LIMIT_KB export INGEST_LATE_EVENT_HOURS INGEST_DEDUP_TTL_HOURS INGEST_WRITE_KEY_CACHE_TTL_SECONDS export INGEST_LOG_PAYLOAD_ON_SUCCESS INGEST_LOG_PAYLOAD_ON_ERROR INGEST_SHUTDOWN_TIMEOUT_SECONDS export KAFKA_TOPIC_INGEST KAFKA_TOPIC_DLQ KAFKA_TOPIC_RETRY export BULKER_HTTP_ADDR BULKER_LOG_LEVEL BULKER_KAFKA_GROUP BULKER_BATCH_SIZE export BULKER_BATCH_INTERVAL_SECONDS BULKER_SHUTDOWN_TIMEOUT_SECONDS export ROTOR_PORT ROTOR_LOG_LEVEL ROTOR_ISOLATE_MEMORY_MB ROTOR_FUNCTION_TIMEOUT_MS # `migrate` CLI: prefer the locally-installed binary if it exists, otherwise # run the official Docker image. Set MIGRATE_BIN to override. MIGRATE_BIN ?= $(shell command -v migrate 2>/dev/null) ifeq ($(MIGRATE_BIN),) MIGRATE = docker run --rm \ -v $(CURDIR)/$(MIGRATIONS_DIR):/migrations \ $(MIGRATE_IMAGE) -path=/migrations -database "$(POSTGRES_DSN_DOCKER)" MIGRATE_CREATE = docker run --rm \ -v $(CURDIR)/$(MIGRATIONS_DIR):/migrations \ $(MIGRATE_IMAGE) create -ext sql -dir /migrations -seq else MIGRATE = $(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" MIGRATE_CREATE = $(MIGRATE_BIN) create -ext sql -dir $(MIGRATIONS_DIR) -seq endif # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- help: @echo "CDP Ingestion - common tasks" @echo "" @echo " make up docker-compose up infra (Postgres, Redis, Kafka, ClickHouse)" @echo " make down docker-compose down" @echo " make logs tail logs" @echo "" @echo " make migrate/new name=X create new PG migration" @echo " make migrate/up apply PG migrations" @echo " make migrate/down rollback one" @echo " make migrate/status migration status" @echo "" @echo " make clickhouse/up apply ClickHouse DDL" @echo " make clickhouse/down drop ClickHouse schema" @echo "" @echo " make run/ingest run ingest service (port 3049)" @echo " make run/bulker run bulker service (port 3042)" @echo " make run/rotor run rotor service (port 3401)" @echo " make run/console run console UI (port 3000)" @echo "" @echo " make test unit tests" @echo " make test/integration integration tests (testcontainers)" # --------------------------------------------------------------------------- # Docker # --------------------------------------------------------------------------- up: docker compose -f infra/docker/docker-compose.yml up -d down: docker compose -f infra/docker/docker-compose.yml down logs: docker compose -f infra/docker/docker-compose.yml logs -f --tail=200 # --------------------------------------------------------------------------- # PostgreSQL migrations # --------------------------------------------------------------------------- migrate/new: @if [ -z "$(name)" ]; then echo "usage: make migrate/new name=add_xxx"; exit 1; fi $(MIGRATE_CREATE) $(name) migrate/up: $(MIGRATE) up migrate/down: $(MIGRATE) down 1 migrate/status: $(MIGRATE) version # --------------------------------------------------------------------------- # ClickHouse DDL # --------------------------------------------------------------------------- clickhouse/up: @bash infra/scripts/clickhouse_apply.sh up clickhouse/down: @bash infra/scripts/clickhouse_apply.sh down # --------------------------------------------------------------------------- # Run services # --------------------------------------------------------------------------- run/ingest: cd ingest && go run ./cmd/server run/bulker: @echo ">>> CLICKHOUSE_ADDR=$$CLICKHOUSE_ADDR CLICKHOUSE_SECURE=$$CLICKHOUSE_SECURE" cd bulker && go run ./cmd/server debug/env: @echo "CLICKHOUSE_ADDR=$$CLICKHOUSE_ADDR" @echo "CLICKHOUSE_SECURE=$$CLICKHOUSE_SECURE" @echo "POSTGRES_DSN=$$POSTGRES_DSN" @echo "-- child env --" @env | grep -E '^(CLICKHOUSE|POSTGRES|KAFKA|INGEST|BULKER)' | sort run/rotor: @cd rotor && [ -d node_modules ] || npm install cd rotor && npm run dev run/console: @cd console && [ -d node_modules ] || npm install cd console && npm run dev # --------------------------------------------------------------------------- # Build # --------------------------------------------------------------------------- build/ingest: cd ingest && CGO_ENABLED=0 go build -o ../bin/ingest ./cmd/server build/bulker: cd bulker && CGO_ENABLED=0 go build -o ../bin/bulker ./cmd/server # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- test: cd ingest && go test ./... -count=1 cd bulker && go test ./... -count=1 test/integration: cd ingest && go test -tags=integration ./... -count=1 -timeout=5m cd bulker && go test -tags=integration ./... -count=1 -timeout=5m # --------------------------------------------------------------------------- # Code quality # --------------------------------------------------------------------------- lint: cd ingest && golangci-lint run ./... cd bulker && golangci-lint run ./... fmt: cd ingest && gofmt -w . cd bulker && gofmt -w . tidy: cd ingest && go mod tidy cd bulker && go mod tidy