BEAM-13939: Restructure Protos to fix namespace conflicts (#16961)

* Updates the Beam model protos to be housed in a namespaced directory structure. This is so that the proto files require usage of a org/apache/beam/model namespace in their imports and so that the generated files also include this namespace in their source file metadata.

* Updates session.proto to use the new namespaced import

* Adds a generic proto generated script that can be used to correctly generate all proto files for the go sdk. This new tool will add any necessary options to the proto compiler and generate the proto files relative to the go sdk root to ensure that the generated files have a namespaced file path in their metadata. If you want to generate a proto file in the go sdk, simply use this script in the go:generate directive, the rest will be taken care of by the script.

* Updates the go sdk to use the new genproto script to correctly generate proto bindings. Updates the README for how to generate the model proto bindings into the SDK

* Regenerates all go sdk proto bindings

* Updates to the python build system. gen_protos.py now supports the new namespaced structure of the Beam model. It does this by supporting arbitrary directory structures of proto files by calculating and replacing the generated imports with relative imports with the generated source. Additionally, it will generate bindings that allow for imports of the form `from apache_beam.portability.api import beam_runner_api_pb2` so that the SDK is not dependent on the potentially changing structure of the generated bindings within `api`. Imports of the form `from apache_beam.portability.api.org.apache.beam.model import beam_runner_api_pb2` are still supported. setup.py now attempts to generate the proto bindings on invocation since the package structure must exist before the wheel can be created.

* Updates to various build, ignore, and linting tooling for python in order to support the new python output structure

* Updates to the python SDK to use the new import structure created by gen_protos

.gitignore

@@ -51,8 +51,7 @@ sdks/python/**/*.egg
 sdks/python/LICENSE
 sdks/python/NOTICE
 sdks/python/README.md
-sdks/python/apache_beam/portability/api/*pb2*.*
-sdks/python/apache_beam/portability/api/*.yaml
+sdks/python/apache_beam/portability/api/*
 sdks/python/nosetests*.xml
 sdks/python/pytest*.xml
 sdks/python/postcommit_requirements.txt

build.gradle.kts

@@ -57,7 +57,7 @@ tasks.rat {
     // Default eclipse excludes neglect subprojects
 
     // Proto/grpc generated wrappers
-    "**/apache_beam/portability/api/*_pb2*.py",
+    "**/apache_beam/portability/api/**/*_pb2*.py",
     "**/go/pkg/beam/**/*.pb.go",
 
     // Ignore go.sum files, which don't permit headers

model/fn-execution/src/main/proto/beam_fn_api.proto → model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto

@@ -38,12 +38,12 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecutio
 option java_package = "org.apache.beam.model.fnexecution.v1";
 option java_outer_classname = "BeamFnApi";
 
-import "beam_runner_api.proto";
-import "endpoints.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/endpoints.proto";
+import "org/apache/beam/model/pipeline/v1/metrics.proto";
 import "google/protobuf/descriptor.proto";
 import "google/protobuf/timestamp.proto";
 import "google/protobuf/duration.proto";
-import "metrics.proto";
 
 // A descriptor for connecting to a remote port using the Beam Fn Data API.
 // Allows for communication between two environments (for example between the

model/fn-execution/src/main/proto/beam_provision_api.proto → model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_provision_api.proto

@@ -29,8 +29,8 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecutio
 option java_package = "org.apache.beam.model.fnexecution.v1";
 option java_outer_classname = "ProvisionApi";
 
-import "beam_runner_api.proto";
-import "endpoints.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/endpoints.proto";
 import "google/protobuf/struct.proto";
 
 // A service to provide runtime provisioning information to the SDK harness

model/interactive/src/main/proto/beam_interactive_api.proto → model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto

@@ -32,7 +32,7 @@ option go_package = "interactive_v1";
 option java_package = "org.apache.beam.model.interactive.v1";
 option java_outer_classname = "InteractiveApi";
 
-import "beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
 import "google/protobuf/timestamp.proto";
 
 // The first record. This contains metadata about the stream and how to

model/job-management/src/main/proto/beam_artifact_api.proto → model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto

@@ -29,7 +29,7 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagem
 option java_package = "org.apache.beam.model.jobmanagement.v1";
 option java_outer_classname = "ArtifactApi";
 
-import "beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
 
 // A service to retrieve artifacts for use in a Job.
 service ArtifactRetrievalService {

model/job-management/src/main/proto/beam_expansion_api.proto → model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto

@@ -29,7 +29,7 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagem
 option java_package = "org.apache.beam.model.expansion.v1";
 option java_outer_classname = "ExpansionApi";
 
-import "beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
 
 message ExpansionRequest {
   // Set of components needed to interpret the transform, or which

model/job-management/src/main/proto/beam_job_api.proto → model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto

@@ -29,11 +29,11 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagem
 option java_package = "org.apache.beam.model.jobmanagement.v1";
 option java_outer_classname = "JobApi";
 
-import "beam_runner_api.proto";
-import "endpoints.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/endpoints.proto";
+import "org/apache/beam/model/pipeline/v1/metrics.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
-import "metrics.proto";
 
 // Job Service for running RunnerAPI pipelines
 service JobService {

model/pipeline/src/main/proto/beam_runner_api.proto → model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto

@@ -29,7 +29,7 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v
 option java_package = "org.apache.beam.model.pipeline.v1";
 option java_outer_classname = "RunnerApi";
 
-import "endpoints.proto";
+import "org/apache/beam/model/pipeline/v1/endpoints.proto";
 import "google/protobuf/descriptor.proto";
 
 message BeamConstants {

model/pipeline/src/main/proto/external_transforms.proto → model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto

@@ -28,8 +28,8 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v
 option java_package = "org.apache.beam.model.pipeline.v1";
 option java_outer_classname = "ExternalTransforms";
 
-import "schema.proto";
-import "beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/schema.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
 
 // A configuration payload for an external transform.
 // Used as the payload of ExternalTransform as part of an ExpansionRequest.

model/pipeline/src/main/proto/metrics.proto → model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto

@@ -29,7 +29,7 @@ option java_package = "org.apache.beam.model.pipeline.v1";
 option java_outer_classname = "MetricsApi";
 
 
-import "beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
 import "google/protobuf/descriptor.proto";
 import "google/protobuf/timestamp.proto";
 

model/pipeline/src/main/proto/standard_window_fns.proto → model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/standard_window_fns.proto

@@ -29,7 +29,7 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v
 option java_package = "org.apache.beam.model.pipeline.v1";
 option java_outer_classname = "StandardWindowFns";
 
-import "beam_runner_api.proto";
+import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
 

sdks/go/pkg/beam/core/runtime/graphx/v1/gen.go

@@ -15,4 +15,4 @@
 
 package v1
 
-//go:generate protoc -I . v1.proto --go_out=../../../../../../../../../../../
+//go:generate ../../../../../../scripts/genproto.sh

sdks/go/pkg/beam/core/runtime/harness/gen.go

@@ -15,4 +15,4 @@
 
 package harness
 
-//go:generate protoc -I. -I../../../../../../../model/fn-execution/src/main/proto -I../../../../../../../model/pipeline/src/main/proto session.proto --go_out=Mbeam_fn_api.proto=github.com/apache/beam/sdks/go/pkg/beam/model/fnexecution_v1:../../../../../../../../../../
+//go:generate ../../../../../scripts/genproto.sh

sdks/go/pkg/beam/core/runtime/harness/session.proto

@@ -26,7 +26,7 @@ package org.apache.beam.sdks.go.pkg.beam.core.runtime.harness.session;
 
 option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/harness/session;session";
 
-import "beam_fn_api.proto";
+import "org/apache/beam/model/fn_execution/v1/beam_fn_api.proto";
 
 message Header {
   string version = 1; // version identifier of the session schema

sdks/go/pkg/beam/model/PROTOBUF.md

@@ -30,19 +30,15 @@ First, follow this one-time setup:
 1. Download [the protobuf compiler](https://github.com/google/protobuf/releases).
    The simplest approach is to download one of the prebuilt binaries (named
    `protoc`) and extract it somewhere in your machine's `$PATH`.
-1. Add `$GOBIN` to your `$PATH`. (Note: If `$GOBIN` is not set, add `$GOPATH/bin`
-   instead.)
 
 To generate the code:
 
 1. Navigate to this directory (`pkg/beam/model`).
-1. Check [go.mod](https://github.com/apache/beam/blob/master/sdks/go.mod) and
+2. Check [go.mod](https://github.com/apache/beam/blob/master/sdks/go.mod) and
    make note of which versions of [google.golang.org/protobuf](https://github.com/protocolbuffers/protobuf-go)
    and [google.golang.org/grpc](https://github.com/grpc/grpc-go) are required.
-1. Install the compiler executables at the corresponding versions.
-    1. `go install google.golang.org/protobuf/cmd/protoc-gen-go@<protobuf_version>`
-    1. `go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@<grpc_version>`
-1. `go generate`
+3. Verify the versions in ../../../scripts/genproto.sh are correct
+4. `go generate`
 
 ## Generated Go code fails to build