138 lines
4.5 KiB
Makefile
138 lines
4.5 KiB
Makefile
.PHONY: help up down logs migrate/new migrate/up migrate/down migrate/status \
|
|
clickhouse/up clickhouse/down \
|
|
run/ingest run/bulker run/rotor run/console \
|
|
build/ingest build/bulker \
|
|
test test/integration \
|
|
lint fmt tidy
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
POSTGRES_DSN ?= postgres://cdp:cdp@localhost:5432/cdp?sslmode=disable
|
|
CLICKHOUSE_DSN ?= clickhouse://default:@localhost:9000/cdp
|
|
MIGRATE_BIN ?= migrate
|
|
MIGRATIONS_DIR := infra/migrations
|
|
CLICKHOUSE_DIR := infra/clickhouse
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
help:
|
|
@echo "CDP Ingestion - common tasks"
|
|
@echo ""
|
|
@echo " make up docker-compose up infra (Postgres, Redis, Kafka, ClickHouse)"
|
|
@echo " make down docker-compose down"
|
|
@echo " make logs tail logs"
|
|
@echo ""
|
|
@echo " make migrate/new name=X create new PG migration"
|
|
@echo " make migrate/up apply PG migrations"
|
|
@echo " make migrate/down rollback one"
|
|
@echo " make migrate/status migration status"
|
|
@echo ""
|
|
@echo " make clickhouse/up apply ClickHouse DDL"
|
|
@echo " make clickhouse/down drop ClickHouse schema"
|
|
@echo ""
|
|
@echo " make run/ingest run ingest service (port 3049)"
|
|
@echo " make run/bulker run bulker service (port 3042)"
|
|
@echo " make run/rotor run rotor service (port 3401)"
|
|
@echo " make run/console run console UI (port 3000)"
|
|
@echo ""
|
|
@echo " make test unit tests"
|
|
@echo " make test/integration integration tests (testcontainers)"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Docker
|
|
# ---------------------------------------------------------------------------
|
|
|
|
up:
|
|
docker compose -f infra/docker/docker-compose.yml up -d
|
|
|
|
down:
|
|
docker compose -f infra/docker/docker-compose.yml down
|
|
|
|
logs:
|
|
docker compose -f infra/docker/docker-compose.yml logs -f --tail=200
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PostgreSQL migrations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
migrate/new:
|
|
@if [ -z "$(name)" ]; then echo "usage: make migrate/new name=add_xxx"; exit 1; fi
|
|
$(MIGRATE_BIN) create -ext sql -dir $(MIGRATIONS_DIR) -seq $(name)
|
|
|
|
migrate/up:
|
|
$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" up
|
|
|
|
migrate/down:
|
|
$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" down 1
|
|
|
|
migrate/status:
|
|
$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" version
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ClickHouse DDL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
clickhouse/up:
|
|
@bash infra/scripts/clickhouse_apply.sh up
|
|
|
|
clickhouse/down:
|
|
@bash infra/scripts/clickhouse_apply.sh down
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Run services
|
|
# ---------------------------------------------------------------------------
|
|
|
|
run/ingest:
|
|
cd ingest && go run ./cmd/server
|
|
|
|
run/bulker:
|
|
cd bulker && go run ./cmd/server
|
|
|
|
run/rotor:
|
|
cd rotor && npm run dev
|
|
|
|
run/console:
|
|
cd console && npm run dev
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Build
|
|
# ---------------------------------------------------------------------------
|
|
|
|
build/ingest:
|
|
cd ingest && CGO_ENABLED=0 go build -o ../bin/ingest ./cmd/server
|
|
|
|
build/bulker:
|
|
cd bulker && CGO_ENABLED=0 go build -o ../bin/bulker ./cmd/server
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
test:
|
|
cd ingest && go test ./... -count=1
|
|
cd bulker && go test ./... -count=1
|
|
|
|
test/integration:
|
|
cd ingest && go test -tags=integration ./... -count=1 -timeout=5m
|
|
cd bulker && go test -tags=integration ./... -count=1 -timeout=5m
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Code quality
|
|
# ---------------------------------------------------------------------------
|
|
|
|
lint:
|
|
cd ingest && golangci-lint run ./...
|
|
cd bulker && golangci-lint run ./...
|
|
|
|
fmt:
|
|
cd ingest && gofmt -w .
|
|
cd bulker && gofmt -w .
|
|
|
|
tidy:
|
|
cd ingest && go mod tidy
|
|
cd bulker && go mod tidy
|