From bc56a61fc4f0cdf5d700a47e6170b5304584d912 Mon Sep 17 00:00:00 2001
From: Igor Drozdov <idrozdov@gitlab.com>
Date: Mon, 7 Jun 2021 09:45:29 +0300
Subject: [PATCH] Resolve Yajl encoding incompatibility
When Yajl encoder was provided a string with ASCII-8BIT,
"incompatible character encodings: ASCII-8BIT and UTF-8" error
was raised.
Let's use StringIO in order to build the result out of chunks.
Changelog: fixed
MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63379
---
lib/gitlab/json.rb | 4 ++--
spec/lib/gitlab/json_spec.rb | 12 +++++++++++-
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index 561cd4509b1a..767ce310b5a0 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -242,7 +242,7 @@ class LimitedEncoder
def self.encode(object, limit: 25.megabytes)
return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder)
- buffer = []
+ buffer = StringIO.new
buffer_size = 0
::Yajl::Encoder.encode(object) do |data_chunk|
@@ -254,7 +254,7 @@ def self.encode(object, limit: 25.megabytes)
buffer_size += chunk_size
end
- buffer.join('')
+ buffer.string
end
end
end
diff --git a/spec/lib/gitlab/json_spec.rb b/spec/lib/gitlab/json_spec.rb
index 42c4b315edf0..f9f57752b0ab 100644
--- a/spec/lib/gitlab/json_spec.rb
+++ b/spec/lib/gitlab/json_spec.rb
@@ -411,7 +411,7 @@
end
describe Gitlab::Json::LimitedEncoder do
- subject { described_class.encode(obj, limit: 8.kilobytes) }
+ subject { described_class.encode(obj, limit: 10.kilobytes) }
context 'when object size is acceptable' do
let(:obj) { { test: true } }
@@ -431,6 +431,16 @@
end
end
+ context 'when object contains ASCII-8BIT encoding' do
+ let(:obj) { [{ a: "\x8F" }] * 1000 }
+
+ it 'does not raise encoding error' do
+ expect { subject }.not_to raise_error
+ expect(subject).to be_a(String)
+ expect(subject.size).to eq(10001)
+ end
+ end
+
context 'when json_limited_encoder is disabled' do
let(:obj) { [{ test: true }] * 1000 }
--
GitLab