Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ load("@gazelle//:def.bzl", "gazelle")
# Resolve protobuf import ambiguities - use the actual protopb packages, not the proto aliases
# gazelle:resolve go github.com/uber/submitqueue/api/base/change/protopb //api/base/change/protopb
# gazelle:resolve go github.com/uber/submitqueue/api/base/mergestrategy/protopb //api/base/mergestrategy/protopb
# gazelle:resolve go github.com/uber/submitqueue/api/runway/messagequeue/protopb //api/runway/messagequeue/protopb
# gazelle:resolve go github.com/uber/submitqueue/api/runway/orchestrator/protopb //api/runway/orchestrator/protopb
# gazelle:resolve go github.com/uber/submitqueue/api/submitqueue/gateway/protopb //api/submitqueue/gateway/protopb
# gazelle:resolve go github.com/uber/submitqueue/api/submitqueue/orchestrator/protopb //api/submitqueue/orchestrator/protopb
# gazelle:resolve go github.com/uber/submitqueue/api/stovepipe/gateway/protopb //api/stovepipe/gateway/protopb
Expand Down
51 changes: 48 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ STOVEPIPE_STACK_COMPOSE_FILE = example/stovepipe/docker-compose.yml
# Fixed project name for local manual testing (tests use unique random names)
STOVEPIPE_LOCAL_PROJECT = stovepipe

# Runway compose files
RUNWAY_ORCHESTRATOR_COMPOSE_FILE = example/runway/orchestrator/server/docker-compose.yml

# Fixed project name for local manual testing (tests use unique random names)
RUNWAY_LOCAL_PROJECT = runway

# yamlfmt version for YAML formatting (override with: make fmt YAMLFMT_VERSION=v0.16.0)
YAMLFMT_VERSION ?= v0.16.0

Expand All @@ -31,7 +37,7 @@ GOIMPORTS_VERSION ?= v0.33.0
# (the out_dir convention in tool/proto/BUILD.bazel) and copied back here. A
# package may hold multiple .proto files (e.g. an RPC contract plus messagequeue
# contracts); all generated stubs land in the same protopb/ dir.
PROTO_PACKAGES = api/base/change api/base/mergestrategy api/base/messagequeue api/runway/messagequeue api/submitqueue/gateway api/submitqueue/orchestrator api/stovepipe/gateway api/stovepipe/orchestrator
PROTO_PACKAGES = api/base/change api/base/mergestrategy api/base/messagequeue api/runway/messagequeue api/runway/orchestrator api/submitqueue/gateway api/submitqueue/orchestrator api/stovepipe/gateway api/stovepipe/orchestrator

# Set REPO_ROOT for docker-compose
export REPO_ROOT := $(shell pwd)
Expand All @@ -47,7 +53,7 @@ define assert_clean
fi
endef

.PHONY: build build-all-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-init-stovepipe-queue-schema local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-gateway-start local-stovepipe-orchestrator-start local-stovepipe-start mocks proto query-deps query-targets run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe-gateway run-client-stovepipe-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help
.PHONY: build build-all-linux build-runway-orchestrator-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-init-runway-queue-schema local-runway-orchestrator-start local-runway-orchestrator-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-init-stovepipe-queue-schema local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-gateway-start local-stovepipe-orchestrator-start local-stovepipe-start mocks proto query-deps query-targets run-client-runway-orchestrator run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe-gateway run-client-stovepipe-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help


build: ## Build all services and examples
Expand All @@ -56,9 +62,17 @@ build: ## Build all services and examples
@echo "Build complete!"

# Build Linux binaries required for Docker containers
build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux ## Build all Linux binaries for Docker
build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux build-runway-orchestrator-linux ## Build all Linux binaries for Docker
@echo "All Linux binaries ready for Docker"

build-runway-orchestrator-linux: ## Build Runway orchestrator Linux binary for Docker
@echo "Building Runway orchestrator Linux binary for Docker..."
@$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/runway/orchestrator/server:orchestrator
@mkdir -p .docker-bin
@cp -f bazel-bin/example/runway/orchestrator/server/orchestrator_/orchestrator .docker-bin/runway-orchestrator 2>/dev/null || \
cp -f bazel-bin/example/runway/orchestrator/server/orchestrator .docker-bin/runway-orchestrator
@echo "Runway orchestrator Linux binary ready at .docker-bin/runway-orchestrator"

build-submitqueue-gateway-linux: ## Build Gateway Linux binary for Docker
@echo "Building Gateway Linux binary for Docker..."
@$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/submitqueue/gateway/server:gateway
Expand Down Expand Up @@ -217,6 +231,32 @@ local-init-stovepipe-queue-schema: ## Apply queue schema only (mysql-queue) for
done
@echo "✅ Stovepipe queue schema applied successfully"

local-init-runway-queue-schema: ## Apply queue schema only (mysql-queue) for Runway compose stacks
@echo "Applying queue schema to mysql-queue (Runway; consumes the merge queues)..."
@for file in platform/extension/messagequeue/mysql/schema/*.sql; do \
echo " - Applying $$(basename $$file)..."; \
docker exec -i $(RUNWAY_LOCAL_PROJECT)-mysql-queue-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \
done
@echo "✅ Runway queue schema applied successfully"

local-runway-orchestrator-start: build-runway-orchestrator-linux ## Start Runway orchestrator locally (orchestrator + MySQL queue)
@echo "Starting Runway orchestrator with compose..."
@$(COMPOSE) -f $(RUNWAY_ORCHESTRATOR_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) up -d --build --wait
@echo "Applying queue schema to mysql-queue (no Runway app schema)..."
@$(MAKE) -s local-init-runway-queue-schema
@echo ""
@echo "✅ Runway orchestrator is running!"
@echo ""
@$(COMPOSE) -f $(RUNWAY_ORCHESTRATOR_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) ps
@echo ""
@echo "Runway orchestrator gRPC port: $$(docker port $(RUNWAY_LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
@echo "MySQL Queue port: $$(docker port $(RUNWAY_LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')"

local-runway-orchestrator-stop: ## Stop Runway orchestrator service
@echo "Stopping Runway orchestrator services..."
@$(COMPOSE) -f $(RUNWAY_ORCHESTRATOR_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) down
@echo "Runway orchestrator services stopped."

local-submitqueue-logs: ## View logs from all running services
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) logs -f

Expand Down Expand Up @@ -282,6 +322,7 @@ local-stop: ## Stop all services (keep data)
@echo "Stopping all services..."
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(RUNWAY_ORCHESTRATOR_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) down
@echo "Services stopped. Data volumes preserved."

local-stovepipe-logs: ## View logs from all running Stovepipe services
Expand Down Expand Up @@ -378,6 +419,10 @@ run-client-stovepipe-gateway:
run-client-stovepipe-orchestrator:
@$(BAZEL) run //example/stovepipe/orchestrator/client:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8084) -message "$(or $(MESSAGE),ping)"

# Run runway orchestrator client (connects to any running runway orchestrator service)
run-client-runway-orchestrator:
@$(BAZEL) run //example/runway/orchestrator/client:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8086) -message "$(or $(MESSAGE),ping)"

run-queue-admin: ## Run queue-admin CLI (use ARGS to pass arguments, e.g. make run-queue-admin ARGS="list-topics")
@$(BAZEL) run //platform/extension/messagequeue/mysql/ctl -- $(ARGS)

Expand Down
1 change: 1 addition & 0 deletions api/runway/messagequeue/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ go_test(
deps = [
"//api/base/change/protopb",
"//api/base/mergestrategy/protopb",
"//api/runway/messagequeue/protopb",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@org_golang_google_protobuf//proto",
Expand Down
40 changes: 40 additions & 0 deletions api/runway/orchestrator/proto/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
load("@rules_go//go:def.bzl", "go_library")
load("@rules_go//proto:def.bzl", "go_proto_library")
load("@rules_proto//proto:defs.bzl", "proto_library")

exports_files(
["orchestrator.proto"],
visibility = ["//tool/proto:__pkg__"],
)

proto_library(
name = "orchestratorpb_proto",
srcs = ["orchestrator.proto"],
visibility = ["//visibility:public"],
)

# keep
go_proto_library(
name = "orchestratorpb_go_proto",
compilers = [
"@rules_go//proto:go_proto",
"@rules_go//proto:go_grpc_v2",
],
importpath = "github.com/uber/submitqueue/api/runway/orchestrator/proto",
proto = ":orchestratorpb_proto",
visibility = ["//visibility:public"],
)

go_library(
name = "proto",
embed = [":orchestratorpb_go_proto"],
importpath = "github.com/uber/submitqueue/api/runway/orchestrator/proto",
visibility = ["//visibility:public"],
)

go_library(
name = "protopb",
embed = [":orchestratorpb_go_proto"],
importpath = "github.com/uber/submitqueue/api/runway/orchestrator/protopb",
visibility = ["//visibility:public"],
)
46 changes: 46 additions & 0 deletions api/runway/orchestrator/proto/orchestrator.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Copyright (c) 2025 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package uber.runway.orchestrator;

option go_package = "github.com/uber/submitqueue/api/runway/orchestrator/protopb";
option java_multiple_files = true;
option java_outer_classname = "OrchestratorProto";
option java_package = "com.uber.submitqueue.runway.orchestrator";

// PingRequest is the request for the Ping method
message PingRequest {
// Optional message to include in the ping
string message = 1;
}

// PingResponse is the response for the Ping method
message PingResponse {
// The response message
string message = 1;
// The service name that handled the request
string service_name = 2;
// Timestamp of when the ping was received
int64 timestamp = 3;
// Hostname of the server that handled the request
string hostname = 4;
}

// RunwayOrchestrator provides the Runway orchestrator API.
service RunwayOrchestrator {
// Ping returns a response indicating the service is alive
rpc Ping(PingRequest) returns (PingResponse) {}
}
26 changes: 26 additions & 0 deletions api/runway/orchestrator/protopb/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
load("@rules_go//go:def.bzl", "go_library")

go_library(
name = "protopb",
srcs = [
"orchestrator.pb.go",
"orchestrator.pb.yarpc.go",
"orchestrator_grpc.pb.go",
],
importpath = "github.com/uber/submitqueue/api/runway/orchestrator/protopb",
visibility = ["//visibility:public"],
deps = [
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
"@org_golang_google_protobuf//proto",
"@org_golang_google_protobuf//reflect/protoreflect",
"@org_golang_google_protobuf//runtime/protoimpl",
"@org_uber_go_fx//:fx",
"@org_uber_go_yarpc//:yarpc",
"@org_uber_go_yarpc//api/transport",
"@org_uber_go_yarpc//api/x/restriction",
"@org_uber_go_yarpc//encoding/protobuf/reflection",
"@org_uber_go_yarpc//encoding/protobuf/v2:protobuf",
],
)
Loading
Loading