data layer

data-layer/Makefile

@@ -0,0 +1,120 @@
+.PHONY: help \
+        migrate/new migrate/up migrate/down migrate/status \
+        clickhouse/up clickhouse/down \
+        run/api run/workers run/console \
+        build/api build/workers \
+        test test/integration \
+        lint fmt tidy
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+POSTGRES_DSN ?= postgres://cdp:cdp@localhost:5432/cdp?sslmode=disable
+CLICKHOUSE_DSN ?= clickhouse://default:@localhost:9000/cdp
+MIGRATE_BIN ?= migrate
+MIGRATIONS_DIR := infra/migrations
+CLICKHOUSE_DIR := infra/clickhouse
+
+# ---------------------------------------------------------------------------
+# Help
+# ---------------------------------------------------------------------------
+
+help:
+	@echo "CDP Analytics (data-layer) - common tasks"
+	@echo ""
+	@echo "  make migrate/new name=X    create new PG migration"
+	@echo "  make migrate/up            apply PG migrations"
+	@echo "  make migrate/down          rollback one"
+	@echo "  make migrate/status        migration status"
+	@echo ""
+	@echo "  make clickhouse/up         apply analytics ClickHouse DDL"
+	@echo "  make clickhouse/down       drop analytics ClickHouse schema"
+	@echo ""
+	@echo "  make run/api               run analytics API   (port 4000)"
+	@echo "  make run/workers           run analytics worker (port 4001)"
+	@echo "  make run/console           run analytics console (port 4002)"
+	@echo ""
+	@echo "  make test                  unit tests"
+	@echo "  make test/integration      integration tests (testcontainers)"
+	@echo ""
+	@echo "  Shared infra (postgres, redis, clickhouse) lives in ../ingestion."
+	@echo "  Run 'make up' from there first."
+
+# ---------------------------------------------------------------------------
+# PostgreSQL migrations
+# ---------------------------------------------------------------------------
+
+migrate/new:
+	@if [ -z "$(name)" ]; then echo "usage: make migrate/new name=add_xxx"; exit 1; fi
+	$(MIGRATE_BIN) create -ext sql -dir $(MIGRATIONS_DIR) -seq $(name)
+
+migrate/up:
+	$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" up
+
+migrate/down:
+	$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" down 1
+
+migrate/status:
+	$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" version
+
+# ---------------------------------------------------------------------------
+# ClickHouse DDL
+# ---------------------------------------------------------------------------
+
+clickhouse/up:
+	@bash infra/scripts/clickhouse_apply.sh up
+
+clickhouse/down:
+	@bash infra/scripts/clickhouse_apply.sh down
+
+# ---------------------------------------------------------------------------
+# Run services
+# ---------------------------------------------------------------------------
+
+run/api:
+	cd api && go run ./cmd/server
+
+run/workers:
+	cd workers && go run ./cmd/worker
+
+run/console:
+	cd console && npm run dev
+
+# ---------------------------------------------------------------------------
+# Build
+# ---------------------------------------------------------------------------
+
+build/api:
+	cd api && CGO_ENABLED=0 go build -o ../bin/api ./cmd/server
+
+build/workers:
+	cd workers && CGO_ENABLED=0 go build -o ../bin/worker ./cmd/worker
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+test:
+	cd api && go test ./... -count=1
+	cd workers && go test ./... -count=1
+
+test/integration:
+	cd api && go test -tags=integration ./... -count=1 -timeout=5m
+	cd workers && go test -tags=integration ./... -count=1 -timeout=5m
+
+# ---------------------------------------------------------------------------
+# Code quality
+# ---------------------------------------------------------------------------
+
+lint:
+	cd api && golangci-lint run ./...
+	cd workers && golangci-lint run ./...
+
+fmt:
+	cd api && gofmt -w .
+	cd workers && gofmt -w .
+
+tidy:
+	cd api && go mod tidy
+	cd workers && go mod tidy