init ingestion

2026-05-24 22:59:24 +07:00
commit 4e8c11d545
80 changed files with 5639 additions and 0 deletions
--- a/ingestion/Makefile
+++ b/ingestion/Makefile
@@ -0,0 +1,137 @@
+.PHONY: help up down logs migrate/new migrate/up migrate/down migrate/status \
+        clickhouse/up clickhouse/down \
+        run/ingest run/bulker run/rotor run/console \
+        build/ingest build/bulker \
+        test test/integration \
+        lint fmt tidy
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+POSTGRES_DSN ?= postgres://cdp:cdp@localhost:5432/cdp?sslmode=disable
+CLICKHOUSE_DSN ?= clickhouse://default:@localhost:9000/cdp
+MIGRATE_BIN ?= migrate
+MIGRATIONS_DIR := infra/migrations
+CLICKHOUSE_DIR := infra/clickhouse
+
+# ---------------------------------------------------------------------------
+# Help
+# ---------------------------------------------------------------------------
+
+help:
+	@echo "CDP Ingestion - common tasks"
+	@echo ""
+	@echo "  make up                    docker-compose up infra (Postgres, Redis, Kafka, ClickHouse)"
+	@echo "  make down                  docker-compose down"
+	@echo "  make logs                  tail logs"
+	@echo ""
+	@echo "  make migrate/new name=X    create new PG migration"
+	@echo "  make migrate/up            apply PG migrations"
+	@echo "  make migrate/down          rollback one"
+	@echo "  make migrate/status        migration status"
+	@echo ""
+	@echo "  make clickhouse/up         apply ClickHouse DDL"
+	@echo "  make clickhouse/down       drop ClickHouse schema"
+	@echo ""
+	@echo "  make run/ingest            run ingest service (port 3049)"
+	@echo "  make run/bulker            run bulker service (port 3042)"
+	@echo "  make run/rotor             run rotor service (port 3401)"
+	@echo "  make run/console           run console UI (port 3000)"
+	@echo ""
+	@echo "  make test                  unit tests"
+	@echo "  make test/integration      integration tests (testcontainers)"
+
+# ---------------------------------------------------------------------------
+# Docker
+# ---------------------------------------------------------------------------
+
+up:
+	docker compose -f infra/docker/docker-compose.yml up -d
+
+down:
+	docker compose -f infra/docker/docker-compose.yml down
+
+logs:
+	docker compose -f infra/docker/docker-compose.yml logs -f --tail=200
+
+# ---------------------------------------------------------------------------
+# PostgreSQL migrations
+# ---------------------------------------------------------------------------
+
+migrate/new:
+	@if [ -z "$(name)" ]; then echo "usage: make migrate/new name=add_xxx"; exit 1; fi
+	$(MIGRATE_BIN) create -ext sql -dir $(MIGRATIONS_DIR) -seq $(name)
+
+migrate/up:
+	$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" up
+
+migrate/down:
+	$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" down 1
+
+migrate/status:
+	$(MIGRATE_BIN) -path $(MIGRATIONS_DIR) -database "$(POSTGRES_DSN)" version
+
+# ---------------------------------------------------------------------------
+# ClickHouse DDL
+# ---------------------------------------------------------------------------
+
+clickhouse/up:
+	@bash infra/scripts/clickhouse_apply.sh up
+
+clickhouse/down:
+	@bash infra/scripts/clickhouse_apply.sh down
+
+# ---------------------------------------------------------------------------
+# Run services
+# ---------------------------------------------------------------------------
+
+run/ingest:
+	cd ingest && go run ./cmd/server
+
+run/bulker:
+	cd bulker && go run ./cmd/server
+
+run/rotor:
+	cd rotor && npm run dev
+
+run/console:
+	cd console && npm run dev
+
+# ---------------------------------------------------------------------------
+# Build
+# ---------------------------------------------------------------------------
+
+build/ingest:
+	cd ingest && CGO_ENABLED=0 go build -o ../bin/ingest ./cmd/server
+
+build/bulker:
+	cd bulker && CGO_ENABLED=0 go build -o ../bin/bulker ./cmd/server
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+test:
+	cd ingest && go test ./... -count=1
+	cd bulker && go test ./... -count=1
+
+test/integration:
+	cd ingest && go test -tags=integration ./... -count=1 -timeout=5m
+	cd bulker && go test -tags=integration ./... -count=1 -timeout=5m
+
+# ---------------------------------------------------------------------------
+# Code quality
+# ---------------------------------------------------------------------------
+
+lint:
+	cd ingest && golangci-lint run ./...
+	cd bulker && golangci-lint run ./...
+
+fmt:
+	cd ingest && gofmt -w .
+	cd bulker && gofmt -w .
+
+tidy:
+	cd ingest && go mod tidy
+	cd bulker && go mod tidy