runable

ingestion/infra/scripts/clickhouse_apply.sh

@@ -5,57 +5,113 @@
 #   clickhouse_apply.sh up      apply *.up.sql in infra/clickhouse/
 #   clickhouse_apply.sh down    apply *.down.sql in REVERSE order
 #
+# Talks to the HTTP(S) interface via curl -- no clickhouse-client binary needed.
+# The SQL is POSTed as the raw body; ClickHouse parses the body when the URL
+# has no ?query= parameter.
+#
 # Env:
-#   CLICKHOUSE_ADDR (default localhost:9000)
-#   CLICKHOUSE_DB   (default cdp)
-#   CLICKHOUSE_USER (default default)
+#   CLICKHOUSE_ADDR     (default localhost:8123)
+#   CLICKHOUSE_DB       (default cdp)
+#   CLICKHOUSE_USER     (default default)
 #   CLICKHOUSE_PASSWORD (default empty)
+#   CLICKHOUSE_SECURE   (default 0; auto-on for port 8443)
 
 set -euo pipefail
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/clickhouse"
-ADDR="${CLICKHOUSE_ADDR:-localhost:9000}"
+ADDR="${CLICKHOUSE_ADDR:-localhost:8123}"
 DB="${CLICKHOUSE_DB:-cdp}"
 USER="${CLICKHOUSE_USER:-default}"
 PASS="${CLICKHOUSE_PASSWORD:-}"
+SECURE="${CLICKHOUSE_SECURE:-0}"
 
 MODE="${1:-up}"
 
 host="${ADDR%%:*}"
 port="${ADDR##*:}"
 
-run_sql() {
+case "$SECURE" in
+    1|true|TRUE|yes) scheme="https" ;;
+    *) [[ "$port" == "8443" ]] && scheme="https" || scheme="http" ;;
+esac
+
+URL_BASE="${scheme}://${host}:${port}/"
+
+# post_sql posts a SQL string to ClickHouse. The query goes in the request
+# body; URL params can carry session options like ?database=foo.
+post_sql() {
+    local sql="$1"
+    local extra_param="${2:-}"
+    local url="$URL_BASE"
+    [[ -n "$extra_param" ]] && url="${URL_BASE}?${extra_param}"
+    printf '%s' "$sql" | curl -sS --fail-with-body \
+        -u "${USER}:${PASS}" \
+        --data-binary @- \
+        "$url"
+    echo
+}
+
+post_sql_file() {
+    # ClickHouse HTTP rejects multi-statements; split the file by `;` and post
+    # each statement individually. Our DDL files use `--` line comments and
+    # do not contain literal `;` inside strings, so a naive RS split is safe.
     local file="$1"
-    echo ">>> applying $(basename "$file")"
-    if [[ -n "$PASS" ]]; then
-        clickhouse-client --host "$host" --port "$port" --user "$USER" --password "$PASS" \
-            --database "$DB" --multiquery --queries-file "$file"
-    else
-        clickhouse-client --host "$host" --port "$port" --user "$USER" \
-            --database "$DB" --multiquery --queries-file "$file"
-    fi
+    local extra_param="${2:-}"
+    local url="$URL_BASE"
+    [[ -n "$extra_param" ]] && url="${URL_BASE}?${extra_param}"
+
+    local tmpdir
+    tmpdir=$(mktemp -d)
+
+    awk -v dir="$tmpdir" '
+        BEGIN { RS=";" ; count=0 }
+        {
+            stmt=$0
+            gsub(/^[[:space:]\n\r]+|[[:space:]\n\r]+$/, "", stmt)
+            if (stmt == "") next
+            count++
+            out=sprintf("%s/%04d.sql", dir, count)
+            print stmt > out
+        }
+    ' "$file"
+
+    local rc=0
+    local s
+    for s in "$tmpdir"/*.sql; do
+        [[ -f "$s" ]] || continue
+        if ! curl -sS --fail-with-body \
+                -u "${USER}:${PASS}" \
+                --data-binary "@${s}" \
+                "$url"; then
+            rc=1
+            break
+        fi
+        echo
+    done
+    rm -rf "$tmpdir"
+    return "$rc"
 }
 
 ensure_db() {
-    if [[ -n "$PASS" ]]; then
-        clickhouse-client --host "$host" --port "$port" --user "$USER" --password "$PASS" \
-            --query "CREATE DATABASE IF NOT EXISTS $DB"
-    else
-        clickhouse-client --host "$host" --port "$port" --user "$USER" \
-            --query "CREATE DATABASE IF NOT EXISTS $DB"
-    fi
+    post_sql "CREATE DATABASE IF NOT EXISTS \`${DB}\`"
+}
+
+run_sql_file() {
+    local file="$1"
+    echo ">>> applying $(basename "$file")"
+    post_sql_file "$file" "database=${DB}"
 }
 
 case "$MODE" in
     up)
         ensure_db
         for f in $(ls "$DIR"/*.up.sql 2>/dev/null | sort); do
-            run_sql "$f"
+            run_sql_file "$f"
         done
         ;;
     down)
         for f in $(ls "$DIR"/*.down.sql 2>/dev/null | sort -r); do
-            run_sql "$f"
+            run_sql_file "$f"
         done
         ;;
     *)