connectrpc: generate code with buf (#4540)

Generate the additional code required for connectrpc with the buf tool.
Currently, there is no bazel ruleset available for buf generation.
However, we can leverage buf as a protoc plugin as shown in
https://github.com/abitofhelp/connect-go-example.
With the help of some convenience macro, source code is copied with our
regular `write_all_source_files` Make target.

This PR is broken out of the
https://github.com/scionproto/scion/tree/connectrpc branch to make
incremental review easier.

Contributes to #4434

BUILD.bazel

@@ -1,5 +1,4 @@
-load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
-load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
 load("@bazel_gazelle//:def.bzl", "gazelle")
 load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@io_bazel_rules_go//go:def.bzl", "nogo")
@@ -130,9 +129,10 @@ gazelle(
 go_lint_config(
     name = "go_lint_config",
     exclude_filter = [
-        "mock_",
-        ".pb.go",
+        ".connect.go",
         ".gen.go",
+        ".pb.go",
+        "mock_",
     ],
     visibility = [
         "//visibility:public",
@@ -250,6 +250,11 @@ write_source_files(
         "//dispatcher/mgmtapi:write_files",
         "//doc/command:write_files",
         "//gateway/mgmtapi:write_files",
+        "//pkg/proto/control_plane/v1/control_planeconnect:write_files",
+        "//pkg/proto/daemon/v1/daemonconnect:write_files",
+        "//pkg/proto/discovery/v1/discoveryconnect:write_files",
+        "//pkg/proto/gateway/v1/gatewayconnect:write_files",
+        "//pkg/proto/hidden_segment/v1/hidden_segmentconnect:write_files",
         "//private/ca/api:write_files",
         "//private/mgmtapi/cppki/api:write_files",
         "//private/mgmtapi/health/api:write_files",

Makefile

@@ -79,7 +79,7 @@ protobuf:
 	rm -f pkg/proto/*/*.pb.go
 	cp -r bazel-bin/pkg/proto/*/go_default_library_/github.com/scionproto/scion/pkg/proto/* pkg/proto
 	cp -r bazel-bin/pkg/proto/*/*/go_default_library_/github.com/scionproto/scion/pkg/proto/* pkg/proto
-	chmod 0644 pkg/proto/*/*.pb.go
+	chmod 0644 pkg/proto/*/*.pb.go pkg/proto/*/*/*.pb.go
 
 mocks:
 	tools/gomocks.py
@@ -142,7 +142,7 @@ lint-protobuf: lint-protobuf-buf
 
 lint-protobuf-buf:
 	$(info ==> $@)
-	@tools/quiet bazel run --config=quiet @buf_bin//file:buf -- check lint
+	@tools/quiet bazel run --config=quiet @buf//:buf -- lint $(PWD) --path $(PWD)/proto
 
 lint-openapi: lint-openapi-spectral
 

WORKSPACE

@@ -2,7 +2,7 @@ workspace(
     name = "com_github_scionproto_scion",
 )
 
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 # linter rules
 http_archive(
@@ -46,6 +46,8 @@ aspect_bazel_lib_register_toolchains()
 # Bazel rules for Golang
 http_archive(
     name = "io_bazel_rules_go",
+    patch_args = ["-p0"],
+    patches = ["//patches:io_bazel_rules_go/import.patch"],
     sha256 = "af47f30e9cbd70ae34e49866e201b3f77069abb111183f2c0297e7e74ba6bbc0",
     urls = [
         "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip",
@@ -204,6 +206,15 @@ load("@tester_debian10_packages//:packages.bzl", tester_debian_packages_install_
 
 tester_debian_packages_install_deps()
 
+# Buf CLI
+http_archive(
+    name = "buf",
+    build_file_content = "exports_files([\"buf\"])",
+    sha256 = "16253b6702dd447ef941b01c9c386a2ab7c8d20bbbc86a5efa5953270f6c9010",
+    strip_prefix = "buf/bin",
+    urls = ["https://github.com/bufbuild/buf/releases/download/v1.32.2/buf-Linux-x86_64.tar.gz"],
+)
+
 # protobuf/gRPC
 http_archive(
     name = "rules_proto_grpc",
@@ -236,16 +247,6 @@ http_archive(
     ],
 )
 
-http_file(
-    name = "buf_bin",
-    downloaded_file_path = "buf",
-    executable = True,
-    sha256 = "5faf15ed0a3cd4bd0919ba5fcb95334c1fd2ba32770df289d615138fa188d36a",
-    urls = [
-        "https://github.com/bufbuild/buf/releases/download/v0.20.5/buf-Linux-x86_64",
-    ],
-)
-
 load("//tools/lint/python:deps.bzl", "python_lint_deps")
 
 python_lint_deps(python_interpreter)

buf.yaml

@@ -1,8 +1,7 @@
-build:
-  roots:
-    - .
-  excludes:
-    - anapaya/protobuf
+version: v1
+breaking:
+  use:
+    - FILE
 lint:
   use:
     - DEFAULT

go.mod

@@ -1,6 +1,7 @@
 module github.com/scionproto/scion
 
 require (
+	connectrpc.com/connect v1.16.2
 	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220209173558-ad29539cd2e9
 	github.com/bazelbuild/rules_go v0.47.1
 	github.com/buildkite/go-buildkite/v2 v2.9.0

go.sum

@@ -1,4 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+connectrpc.com/connect v1.16.2 h1:ybd6y+ls7GOlb7Bh5C8+ghA6SvCBajHwxssO2CGFjqE=
+connectrpc.com/connect v1.16.2/go.mod h1:n2kgwskMHXC+lVqb18wngEpF95ldBHXjZYJussz5FRc=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=

go_deps.bzl

@@ -19,6 +19,12 @@ def go_deps():
         sum = "h1:YGD4H+SuIOOqsyoLOpZDWcieM28W47/zRO7f+9V3nvo=",
         version = "v0.4.5",
     )
+    go_repository(
+        name = "com_connectrpc_connect",
+        importpath = "connectrpc.com/connect",
+        sum = "h1:ybd6y+ls7GOlb7Bh5C8+ghA6SvCBajHwxssO2CGFjqE=",
+        version = "v1.16.2",
+    )
     go_repository(
         name = "com_github_ajstarks_svgo",
         importpath = "github.com/ajstarks/svgo",

patches/BUILD.bazel

@@ -0,0 +1 @@
+exports_files(glob(["**/*.patch"]))

patches/io_bazel_rules_go/import.patch

@@ -0,0 +1,34 @@
+Patch is copied from https://github.com/abitofhelp/connect-go-example/blob/main/import.patch
+
+It is required because of the issue described in https://github.com/bazelbuild/rules_go/issues/3951.
+The connect protoc plugin generates files in a sub-directory, which is not supported by the rules_go.
+This patch allows to override the import path of the generated files.
+
+There is also an upstream discussion whether there should be support for generating files in
+the directory itself rather than in a sub-directory. See:
+https://github.com/connectrpc/connect-go/discussions/310
+
+--- proto/def.bzl
++++ proto/def.bzl
+@@ -123,8 +123,12 @@ def _go_proto_library_impl(ctx):
+             imports = get_imports(ctx.attr, go.importpath),
+             importpath = go.importpath,
+         ))
++    importpath = None
++    if ctx.attr.overrideimportpath:
++        importpath = ctx.attr.overrideimportpath
+     library = go.new_library(
+         go,
++        importpath = importpath,
+         resolver = _proto_library_to_source,
+         srcs = go_srcs,
+     )
+@@ -164,6 +168,7 @@ go_proto_library = rule(
+         "importpath": attr.string(),
+         "importmap": attr.string(),
+         "importpath_aliases": attr.string_list(),  # experimental, undocumented
++        "overrideimportpath": attr.string(),
+         "embed": attr.label_list(providers = [GoLibrary]),
+         "gc_goopts": attr.string_list(),
+         "compiler": attr.label(providers = [GoProtoCompiler]),
+-- 

pkg/proto/control_plane/v1/control_planeconnect/BUILD.bazel

@@ -0,0 +1,12 @@
+load("//tools/proto:connect.bzl", "go_connect_library")
+
+go_connect_library(
+    name = "go_default_library",
+    files = [
+        "cppki.connect.go",
+        "drkey.connect.go",
+        "renewal.connect.go",
+        "seg.connect.go",
+    ],
+    proto = "control_plane",
+)