Git Commit failed with 401 Unauthorized error
Summary
staging环境无法提交代码，报错: pre-receive hooks: GitLab: 401 Unauthorized
似乎gitlab rails 与 gitaly 之间的调用出现权限验证问题。
为了不影响今天的正常使用，staging已经回滚。
版本信息：
Steps to reproduce
由于staging已经回滚，想复现此bug，可以下载安装包
# Ubuntu 20.04
https://devops-downloads-packages-1303695223.cos.ap-shanghai.myqcloud.com/ubuntu-focal/gitlab-jh_15.5.202210140313-d61c8f11f7c_amd64.deb并安装测试。在代码仓库中提交代码即可触发此bug。
Click to expand
Logs
gitlab-rails
{
"_index": "kafka-rails-inf-tstg-000080",
"_type": "_doc",
"_id": "nvUC5IMBf_fojjfkq4vD",
"_version": 1,
"_score": null,
"_source": {
"@timestamp": "2022-10-17T03:35:22.831Z",
"tier": "sv",
"correlation_id": "01GFJ03RSFDC3Y4SX2CQ1H7AG7",
"exception.class": "Gitlab::Git::PreReceiveError",
"exception.message": "401 Unauthorized",
"type": "web",
"tag": "web-rails.var.log.containers.jh-webservice-web-767868b464-fsprv_gitlab_webservice-24a7653258b9515493b9919fb24651430bba87e4b6f5a8dd69e0d4c5cc4635f4.log",
"time": "2022-10-17T03:34:33.409Z",
"input": {
"type": "kafka"
},
"kubernetes": {
"namespace_id": "eefdb3cd-54b4-454e-8d09-55e1852b2ad1",
"pod_id": "b6c57099-5ed9-4f41-8463-00ccfefb7451",
"host": "172.20.32.48",
"labels": {
"chart": "webservice-6.3.0",
"heritage": "Helm",
"pod-template-hash": "767868b464",
"stage": "main",
"gitlab_com/webservice-name": "web",
"app": "webservice",
"deployment": "web",
"release": "jh",
"type": "web"
},
"container_image": "registry.gitlab.cn/cng-images/gitlab-webservice-jh:15-5-202210140313-d61c8f11f7c",
"container_image_id": "registry.gitlab.cn/cng-images/gitlab-webservice-jh@sha256:17a0f4c38f4182b502ae46729872b9dcbc814a23f6438442f768a1b0204b3d1f",
"master_url": "https://10.102.255.1:443/api",
"namespace_labels": {
"name": "gitlab"
},
"container_name": "webservice",
"namespace_name": "gitlab",
"pod_name": "jh-webservice-web-767868b464-fsprv"
},
"tags.correlation_id": "01GFJ03RSFDC3Y4SX2CQ1H7AG7",
"severity": "ERROR",
"exception.backtrace": [
"lib/gitlab/gitaly_client/operation_service.rb:430:in `user_commit_files'",
"lib/gitlab/git/repository.rb:968:in `block in commit_files'",
"lib/gitlab/git/wraps_gitaly_errors.rb:7:in `wrapped_gitaly_errors'",
"lib/gitlab/git/repository.rb:967:in `commit_files'",
"app/models/repository.rb:867:in `block in commit_files'",
"app/models/repository.rb:850:in `with_cache_hooks'",
"app/models/repository.rb:867:in `commit_files'",
"app/models/repository.rb:825:in `create_file'",
"app/services/files/create_service.rb:16:in `create_transformed_commit'",
"app/services/files/create_service.rb:10:in `create_commit!'",
"app/services/commits/create_service.rb:30:in `execute'",
"app/controllers/concerns/creates_commit.rb:29:in `create_commit'",
"app/controllers/projects/blob_controller.rb:55:in `create'",
"ee/lib/gitlab/ip_address_state.rb:10:in `with'",
"ee/app/controllers/ee/application_controller.rb:45:in `set_current_ip_address'",
"app/controllers/application_controller.rb:531:in `set_current_admin'",
"lib/gitlab/session.rb:11:in `with_session'",
"app/controllers/application_controller.rb:522:in `set_session_storage'",
"lib/gitlab/i18n.rb:107:in `with_locale'",
"lib/gitlab/i18n.rb:113:in `with_user_locale'",
"app/controllers/application_controller.rb:516:in `set_locale'",
"app/controllers/application_controller.rb:510:in `set_current_context'",
"ee/lib/omni_auth/strategies/group_saml.rb:41:in `other_phase'",
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
"lib/gitlab/metrics/web_transaction.rb:46:in `run'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
"lib/gitlab/jira/middleware.rb:19:in `call'",
"lib/gitlab/middleware/go.rb:20:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:21:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:26:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:77:in `call'",
"lib/gitlab/middleware/release_env.rb:13:in `call'"
],
"params": null,
"tags.locale": "zh_CN",
"component": "gitlab",
"environment": "tstg",
"tags.stage": "main",
"tags.feature_category": "source_code_management",
"subcomponent": "exceptions_json",
"level": "error",
"stage": "main",
"cluster_name": "tstg-sh2",
"docker": {
"container_id": "24a7653258b9515493b9919fb24651430bba87e4b6f5a8dd69e0d4c5cc4635f4"
},
"user.username": "vincent_stchu",
"kubernetes.region": "ap-shanghai",
"tags.type": "web",
"tags.program": "web",
"shard": "default"
},
"fields": {
"@timestamp": [
"2022-10-17T03:35:22.831Z"
],
"time": [
"2022-10-17T03:34:33.409Z"
]
},
"highlight": {
"correlation_id": [
"@kibana-highlighted-field@01GFJ03RSFDC3Y4SX2CQ1H7AG7@/kibana-highlighted-field@"
]
},
"sort": [
1665977673409
]
}gitaly
{
"_index": "kafka-gitaly-inf-tstg-000079",
"_type": "_doc",
"_id": "bBwC5IMBUt0I9b0IcubH",
"_version": 1,
"_score": null,
"_source": {
"@timestamp": "2022-10-17T03:35:08.249Z",
"grpc.meta.auth_version": "v2",
"start_sha": "",
"grpc.method": "UserCommitFiles",
"correlation_id": "01GFJ03RSFDC3Y4SX2CQ1H7AG7",
"start_branch_name": "bWFpbg==",
"peer.address": "172.20.209.125:56448",
"repository_storage": "gitaly-shards-ssd-1",
"environment": "tstg",
"span.kind": "server",
"tag": "gitaly",
"grpc.meta.deadline_type": "regular",
"force": false,
"stage": "main",
"msg": "user commit files failed",
"system": "grpc",
"grpc.meta.method_type": "client_stream",
"username": "vincent_stchu",
"grpc.service": "gitaly.OperationService",
"start_repository_relative_path": "@hashed/a4/40/a440868cf4311953cb45c7ded9360009e1bb77775b6395a3e13aa9ef831794b1.git",
"shard": "unknown",
"grpc.start_time": "2022-10-17T03:34:33.312",
"branch_name": "bWFpbg==",
"pid": 3196662,
"repository_relative_path": "@hashed/a4/40/a440868cf4311953cb45c7ded9360009e1bb77775b6395a3e13aa9ef831794b1.git",
"fqdn": "saas-staging-gitaly-shards-ssd-1",
"tier": "stor",
"type": "gitaly",
"level": "error",
"grpc.meta.client_name": "gitlab-web",
"grpc.request.deadline": "2022-10-17T03:35:28.012",
"error": "update reference: running pre-receive hooks: GitLab: 401 Unauthorized",
"start_repository_storage": "gitaly-shards-ssd-1",
"hostname": "saas-staging-gitaly-shards-ssd-1",
"grpc.request.fullMethod": "/gitaly.OperationService/UserCommitFiles",
"remote_ip": "101.88.245.130",
"time": "2022-10-17T03:34:33.402Z",
"input": {
"type": "kafka"
}
},
"fields": {
"@timestamp": [
"2022-10-17T03:35:08.249Z"
],
"grpc.request.deadline": [
"2022-10-17T03:35:28.012Z"
],
"time": [
"2022-10-17T03:34:33.402Z"
],
"grpc.start_time": [
"2022-10-17T03:34:33.312Z"
]
},
"highlight": {
"correlation_id": [
"@kibana-highlighted-field@01GFJ03RSFDC3Y4SX2CQ1H7AG7@/kibana-highlighted-field@"
]
},
"sort": [
1665977673402
]
}