190 lines
6.8 KiB
Makefile
190 lines
6.8 KiB
Makefile
.PHONY: help up down logs migrate/new migrate/up migrate/down migrate/status \
|
|
clickhouse/up clickhouse/down \
|
|
run/ingest run/bulker run/rotor run/console \
|
|
build/ingest build/bulker \
|
|
test test/integration \
|
|
lint fmt tidy
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Auto-load .env if present, then export every variable so child processes
|
|
# (go run, docker run, the apply script) inherit them. GNU Make 3.81 (macOS
|
|
# default) ignores bare `export`, so we list the keys explicitly below after
|
|
# all variables have been declared.
|
|
ifneq (,$(wildcard ./.env))
|
|
include .env
|
|
endif
|
|
|
|
POSTGRES_DSN ?= postgres://renolation:renolation@103.188.82.191:5432/ingestion?sslmode=disable
|
|
# For dockerized `migrate`: same DSN works because the target is reachable from
|
|
# the container too (it's a public host, not host.docker.internal).
|
|
POSTGRES_DSN_DOCKER ?= $(POSTGRES_DSN)
|
|
CLICKHOUSE_ADDR ?= 192.168.1.60:8123
|
|
CLICKHOUSE_DB ?= analytics
|
|
CLICKHOUSE_USER ?= renolation
|
|
CLICKHOUSE_PASSWORD ?= renolation
|
|
CLICKHOUSE_SECURE ?= 0
|
|
MIGRATE_IMAGE ?= migrate/migrate:v4.17.1
|
|
MIGRATIONS_DIR := infra/migrations
|
|
CLICKHOUSE_DIR := infra/clickhouse
|
|
|
|
# Re-export everything that the Go services and the apply script read.
|
|
# Listed explicitly so this works on GNU Make 3.81 (macOS default).
|
|
export POSTGRES_DSN REDIS_ADDR KAFKA_BROKERS
|
|
export CLICKHOUSE_ADDR CLICKHOUSE_DB CLICKHOUSE_USER CLICKHOUSE_PASSWORD CLICKHOUSE_SECURE
|
|
export INGEST_HTTP_ADDR INGEST_LOG_LEVEL INGEST_PAYLOAD_LIMIT_KB INGEST_BATCH_LIMIT_KB
|
|
export INGEST_LATE_EVENT_HOURS INGEST_DEDUP_TTL_HOURS INGEST_WRITE_KEY_CACHE_TTL_SECONDS
|
|
export INGEST_LOG_PAYLOAD_ON_SUCCESS INGEST_LOG_PAYLOAD_ON_ERROR INGEST_SHUTDOWN_TIMEOUT_SECONDS
|
|
export KAFKA_TOPIC_INGEST KAFKA_TOPIC_DLQ KAFKA_TOPIC_RETRY
|
|
export BULKER_HTTP_ADDR BULKER_LOG_LEVEL BULKER_KAFKA_GROUP BULKER_BATCH_SIZE
|
|
export BULKER_BATCH_INTERVAL_SECONDS BULKER_SHUTDOWN_TIMEOUT_SECONDS
|
|
export ROTOR_PORT ROTOR_LOG_LEVEL ROTOR_ISOLATE_MEMORY_MB ROTOR_FUNCTION_TIMEOUT_MS
|
|
|
|
# `migrate` CLI: prefer the locally-installed binary if it exists, otherwise
|
|
# run the official Docker image. Set MIGRATE_BIN to override.
|
|
MIGRATE_BIN ?= $(shell command -v migrate 2>/dev/null)
|
|
ifeq ($(MIGRATE_BIN),)
|
|
MIGRATE = docker run --rm \
|
|
-v $(CURDIR)/$(MIGRATIONS_DIR):/migrations \
|
|
$(MIGRATE_IMAGE) -path=/migrations -database "$(POSTGRES_DSN_DOCKER)"
|
|
MIGRATE_CREATE = docker run --rm \
|
|
-v $(CURDIR)/$(MIGRATIONS_DIR):/migrations \
|
|
$(MIGRATE_IMAGE) create -ext sql -dir /migrations -seq
|
|
else
|
|
MIGRATE = $(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)"
|
|
MIGRATE_CREATE = $(MIGRATE_BIN) create -ext sql -dir $(MIGRATIONS_DIR) -seq
|
|
endif
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
help:
|
|
@echo "CDP Ingestion - common tasks"
|
|
@echo ""
|
|
@echo " make up docker-compose up infra (Postgres, Redis, Kafka, ClickHouse)"
|
|
@echo " make down docker-compose down"
|
|
@echo " make logs tail logs"
|
|
@echo ""
|
|
@echo " make migrate/new name=X create new PG migration"
|
|
@echo " make migrate/up apply PG migrations"
|
|
@echo " make migrate/down rollback one"
|
|
@echo " make migrate/status migration status"
|
|
@echo ""
|
|
@echo " make clickhouse/up apply ClickHouse DDL"
|
|
@echo " make clickhouse/down drop ClickHouse schema"
|
|
@echo ""
|
|
@echo " make run/ingest run ingest service (port 3049)"
|
|
@echo " make run/bulker run bulker service (port 3042)"
|
|
@echo " make run/rotor run rotor service (port 3401)"
|
|
@echo " make run/console run console UI (port 3000)"
|
|
@echo ""
|
|
@echo " make test unit tests"
|
|
@echo " make test/integration integration tests (testcontainers)"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Docker
|
|
# ---------------------------------------------------------------------------
|
|
|
|
up:
|
|
docker compose -f infra/docker/docker-compose.yml up -d
|
|
|
|
down:
|
|
docker compose -f infra/docker/docker-compose.yml down
|
|
|
|
logs:
|
|
docker compose -f infra/docker/docker-compose.yml logs -f --tail=200
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PostgreSQL migrations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
migrate/new:
|
|
@if [ -z "$(name)" ]; then echo "usage: make migrate/new name=add_xxx"; exit 1; fi
|
|
$(MIGRATE_CREATE) $(name)
|
|
|
|
migrate/up:
|
|
$(MIGRATE) up
|
|
|
|
migrate/down:
|
|
$(MIGRATE) down 1
|
|
|
|
migrate/status:
|
|
$(MIGRATE) version
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ClickHouse DDL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
clickhouse/up:
|
|
@bash infra/scripts/clickhouse_apply.sh up
|
|
|
|
clickhouse/down:
|
|
@bash infra/scripts/clickhouse_apply.sh down
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Run services
|
|
# ---------------------------------------------------------------------------
|
|
|
|
run/ingest:
|
|
cd ingest && go run ./cmd/server
|
|
|
|
run/bulker:
|
|
@echo ">>> CLICKHOUSE_ADDR=$$CLICKHOUSE_ADDR CLICKHOUSE_SECURE=$$CLICKHOUSE_SECURE"
|
|
cd bulker && go run ./cmd/server
|
|
|
|
debug/env:
|
|
@echo "CLICKHOUSE_ADDR=$$CLICKHOUSE_ADDR"
|
|
@echo "CLICKHOUSE_SECURE=$$CLICKHOUSE_SECURE"
|
|
@echo "POSTGRES_DSN=$$POSTGRES_DSN"
|
|
@echo "-- child env --"
|
|
@env | grep -E '^(CLICKHOUSE|POSTGRES|KAFKA|INGEST|BULKER)' | sort
|
|
|
|
run/rotor:
|
|
@cd rotor && [ -d node_modules ] || npm install
|
|
cd rotor && npm run dev
|
|
|
|
run/console:
|
|
@cd console && [ -d node_modules ] || npm install
|
|
cd console && npm run dev
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Build
|
|
# ---------------------------------------------------------------------------
|
|
|
|
build/ingest:
|
|
cd ingest && CGO_ENABLED=0 go build -o ../bin/ingest ./cmd/server
|
|
|
|
build/bulker:
|
|
cd bulker && CGO_ENABLED=0 go build -o ../bin/bulker ./cmd/server
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
test:
|
|
cd ingest && go test ./... -count=1
|
|
cd bulker && go test ./... -count=1
|
|
|
|
test/integration:
|
|
cd ingest && go test -tags=integration ./... -count=1 -timeout=5m
|
|
cd bulker && go test -tags=integration ./... -count=1 -timeout=5m
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Code quality
|
|
# ---------------------------------------------------------------------------
|
|
|
|
lint:
|
|
cd ingest && golangci-lint run ./...
|
|
cd bulker && golangci-lint run ./...
|
|
|
|
fmt:
|
|
cd ingest && gofmt -w .
|
|
cd bulker && gofmt -w .
|
|
|
|
tidy:
|
|
cd ingest && go mod tidy
|
|
cd bulker && go mod tidy
|