.PHONY: help up down logs migrate/new migrate/up migrate/down migrate/status \ clickhouse/up clickhouse/down \ run/ingest run/bulker run/rotor run/console \ build/ingest build/bulker \ test test/integration \ lint fmt tidy # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- POSTGRES_DSN ?= postgres://cdp:cdp@localhost:5432/cdp?sslmode=disable CLICKHOUSE_DSN ?= clickhouse://default:@localhost:9000/cdp MIGRATE_BIN ?= migrate MIGRATIONS_DIR := infra/migrations CLICKHOUSE_DIR := infra/clickhouse # --------------------------------------------------------------------------- # Help # --------------------------------------------------------------------------- help: @echo "CDP Ingestion - common tasks" @echo "" @echo " make up docker-compose up infra (Postgres, Redis, Kafka, ClickHouse)" @echo " make down docker-compose down" @echo " make logs tail logs" @echo "" @echo " make migrate/new name=X create new PG migration" @echo " make migrate/up apply PG migrations" @echo " make migrate/down rollback one" @echo " make migrate/status migration status" @echo "" @echo " make clickhouse/up apply ClickHouse DDL" @echo " make clickhouse/down drop ClickHouse schema" @echo "" @echo " make run/ingest run ingest service (port 3049)" @echo " make run/bulker run bulker service (port 3042)" @echo " make run/rotor run rotor service (port 3401)" @echo " make run/console run console UI (port 3000)" @echo "" @echo " make test unit tests" @echo " make test/integration integration tests (testcontainers)" # --------------------------------------------------------------------------- # Docker # --------------------------------------------------------------------------- up: docker compose -f infra/docker/docker-compose.yml up -d down: docker compose -f infra/docker/docker-compose.yml down logs: docker compose -f infra/docker/docker-compose.yml logs -f --tail=200 # --------------------------------------------------------------------------- # PostgreSQL migrations # --------------------------------------------------------------------------- migrate/new: @if [ -z "$(name)" ]; then echo "usage: make migrate/new name=add_xxx"; exit 1; fi $(MIGRATE_BIN) create -ext sql -dir $(MIGRATIONS_DIR) -seq $(name) migrate/up: $(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" up migrate/down: $(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" down 1 migrate/status: $(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" version # --------------------------------------------------------------------------- # ClickHouse DDL # --------------------------------------------------------------------------- clickhouse/up: @bash infra/scripts/clickhouse_apply.sh up clickhouse/down: @bash infra/scripts/clickhouse_apply.sh down # --------------------------------------------------------------------------- # Run services # --------------------------------------------------------------------------- run/ingest: cd ingest && go run ./cmd/server run/bulker: cd bulker && go run ./cmd/server run/rotor: cd rotor && npm run dev run/console: cd console && npm run dev # --------------------------------------------------------------------------- # Build # --------------------------------------------------------------------------- build/ingest: cd ingest && CGO_ENABLED=0 go build -o ../bin/ingest ./cmd/server build/bulker: cd bulker && CGO_ENABLED=0 go build -o ../bin/bulker ./cmd/server # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- test: cd ingest && go test ./... -count=1 cd bulker && go test ./... -count=1 test/integration: cd ingest && go test -tags=integration ./... -count=1 -timeout=5m cd bulker && go test -tags=integration ./... -count=1 -timeout=5m # --------------------------------------------------------------------------- # Code quality # --------------------------------------------------------------------------- lint: cd ingest && golangci-lint run ./... cd bulker && golangci-lint run ./... fmt: cd ingest && gofmt -w . cd bulker && gofmt -w . tidy: cd ingest && go mod tidy cd bulker && go mod tidy