diff --git a/app/assets/javascripts/blame/streaming/index.js b/app/assets/javascripts/blame/streaming/index.js
new file mode 100644
index 0000000000000000000000000000000000000000..a74e01b6423d61f4af532ab4f196030266eac673
--- /dev/null
+++ b/app/assets/javascripts/blame/streaming/index.js
@@ -0,0 +1,56 @@
+import { renderHtmlStreams } from '~/streaming/render_html_streams';
+import { handleStreamedAnchorLink } from '~/streaming/handle_streamed_anchor_link';
+import { createAlert } from '~/flash';
+import { __ } from '~/locale';
+import { rateLimitStreamRequests } from '~/streaming/rate_limit_stream_requests';
+import { toPolyfillReadable } from '~/streaming/polyfills';
+
+export async function renderBlamePageStreams(firstStreamPromise) {
+ const element = document.querySelector('#blame-stream-container');
+
+ if (!element || !firstStreamPromise) return;
+
+ const stopAnchorObserver = handleStreamedAnchorLink(element);
+ const { dataset } = document.querySelector('#blob-content-holder');
+ const totalExtraPages = parseInt(dataset.totalExtraPages, 10);
+ const { pagesUrl } = dataset;
+
+ const remainingStreams = rateLimitStreamRequests({
+ factory: (index) => {
+ const url = new URL(pagesUrl);
+ // page numbers start with 1
+ // the first page is already rendered in the document
+ // the second page is passed with the 'firstStreamPromise'
+ url.searchParams.set('page', index + 3);
+ return fetch(url).then((response) => toPolyfillReadable(response.body));
+ },
+ // we don't want to overload gitaly with concurrent requests
+ // https://gitlab.com/gitlab-org/gitlab/-/issues/391842#note_1281695095
+ // using 5 as a good starting point
+ maxConcurrentRequests: 5,
+ total: totalExtraPages,
+ });
+
+ try {
+ await renderHtmlStreams(
+ [firstStreamPromise.then(toPolyfillReadable), ...remainingStreams],
+ element,
+ );
+ } catch (error) {
+ createAlert({
+ message: __('Blame could not be loaded as a single page.'),
+ primaryButton: {
+ text: __('View blame as separate pages'),
+ clickHandler() {
+ const newUrl = new URL(window.location);
+ newUrl.searchParams.delete('streaming');
+ window.location.href = newUrl;
+ },
+ },
+ });
+ throw error;
+ } finally {
+ stopAnchorObserver();
+ document.querySelector('#blame-stream-loading').remove();
+ }
+}
diff --git a/app/assets/javascripts/pages/projects/blame/show/index.js b/app/assets/javascripts/pages/projects/blame/show/index.js
index 1e4b9de90f26779c5e91772e5c310e1fce8d4062..f0fdd18c828dada8b1e7ebd18535b3f437429283 100644
--- a/app/assets/javascripts/pages/projects/blame/show/index.js
+++ b/app/assets/javascripts/pages/projects/blame/show/index.js
@@ -1,5 +1,10 @@
import initBlob from '~/pages/projects/init_blob';
import redirectToCorrectPage from '~/blame/blame_redirect';
+import { renderBlamePageStreams } from '~/blame/streaming';
-redirectToCorrectPage();
+if (new URLSearchParams(window.location.search).get('streaming')) {
+ renderBlamePageStreams(window.blamePageStream);
+} else {
+ redirectToCorrectPage();
+}
initBlob();
diff --git a/app/assets/javascripts/streaming/chunk_writer.js b/app/assets/javascripts/streaming/chunk_writer.js
new file mode 100644
index 0000000000000000000000000000000000000000..4bbd0a5f8438f29d7629ab2ad694e66d826051a4
--- /dev/null
+++ b/app/assets/javascripts/streaming/chunk_writer.js
@@ -0,0 +1,144 @@
+import { throttle } from 'lodash';
+import { RenderBalancer } from '~/streaming/render_balancer';
+import {
+ BALANCE_RATE,
+ HIGH_FRAME_TIME,
+ LOW_FRAME_TIME,
+ MAX_CHUNK_SIZE,
+ MIN_CHUNK_SIZE,
+ TIMEOUT,
+} from '~/streaming/constants';
+
+const defaultConfig = {
+ balanceRate: BALANCE_RATE,
+ minChunkSize: MIN_CHUNK_SIZE,
+ maxChunkSize: MAX_CHUNK_SIZE,
+ lowFrameTime: LOW_FRAME_TIME,
+ highFrameTime: HIGH_FRAME_TIME,
+ timeout: TIMEOUT,
+};
+
+function concatUint8Arrays(a, b) {
+ const array = new Uint8Array(a.length + b.length);
+ array.set(a, 0);
+ array.set(b, a.length);
+ return array;
+}
+
+// This class is used to write chunks with a balanced size
+// to avoid blocking main thread for too long.
+//
+// A chunk can be:
+// 1. Too small
+// 2. Too large
+// 3. Delayed in time
+//
+// This class resolves all these problems by
+// 1. Splitting or concatenating chunks to met the size criteria
+// 2. Rendering current chunk buffer immediately if enough time has passed
+//
+// The size of the chunk is determined by RenderBalancer,
+// It measures execution time for each chunk write and adjusts next chunk size.
+export class ChunkWriter {
+ buffer = null;
+ decoder = new TextDecoder('utf-8');
+ timeout = null;
+
+ constructor(htmlStream, config) {
+ this.htmlStream = htmlStream;
+
+ const { balanceRate, minChunkSize, maxChunkSize, lowFrameTime, highFrameTime, timeout } = {
+ ...defaultConfig,
+ ...config,
+ };
+
+ // ensure we still render chunks over time if the size criteria is not met
+ this.scheduleAccumulatorFlush = throttle(this.flushAccumulator.bind(this), timeout);
+
+ const averageSize = Math.round((maxChunkSize + minChunkSize) / 2);
+ this.size = Math.max(averageSize, minChunkSize);
+
+ this.balancer = new RenderBalancer({
+ lowFrameTime,
+ highFrameTime,
+ decrease: () => {
+ this.size = Math.round(Math.max(this.size / balanceRate, minChunkSize));
+ },
+ increase: () => {
+ this.size = Math.round(Math.min(this.size * balanceRate, maxChunkSize));
+ },
+ });
+ }
+
+ write(chunk) {
+ this.scheduleAccumulatorFlush.cancel();
+
+ if (this.buffer) {
+ this.buffer = concatUint8Arrays(this.buffer, chunk);
+ } else {
+ this.buffer = chunk;
+ }
+
+ // accumulate chunks until the size is fulfilled
+ if (this.size > this.buffer.length) {
+ this.scheduleAccumulatorFlush();
+ return Promise.resolve();
+ }
+
+ return this.balancedWrite();
+ }
+
+ balancedWrite() {
+ let cursor = 0;
+
+ return this.balancer.render(() => {
+ const chunkPart = this.buffer.subarray(cursor, cursor + this.size);
+ // accumulate chunks until the size is fulfilled
+ // this is a hot path for the last chunkPart of the chunk
+ if (chunkPart.length < this.size) {
+ this.buffer = chunkPart;
+ this.scheduleAccumulatorFlush();
+ return false;
+ }
+
+ this.writeToDom(chunkPart);
+
+ cursor += this.size;
+ if (cursor >= this.buffer.length) {
+ this.buffer = null;
+ return false;
+ }
+ // continue render
+ return true;
+ });
+ }
+
+ writeToDom(chunk, stream = true) {
+ // stream: true allows us to split chunks with multi-part words
+ const decoded = this.decoder.decode(chunk, { stream });
+ this.htmlStream.write(decoded);
+ }
+
+ flushAccumulator() {
+ if (this.buffer) {
+ this.writeToDom(this.buffer);
+ this.buffer = null;
+ }
+ }
+
+ close() {
+ this.scheduleAccumulatorFlush.cancel();
+ if (this.buffer) {
+ // last chunk should have stream: false to indicate the end of the stream
+ this.writeToDom(this.buffer, false);
+ this.buffer = null;
+ }
+ this.htmlStream.close();
+ }
+
+ abort() {
+ this.scheduleAccumulatorFlush.cancel();
+ this.buffer = null;
+ this.htmlStream.abort();
+ }
+}
diff --git a/app/assets/javascripts/streaming/constants.js b/app/assets/javascripts/streaming/constants.js
new file mode 100644
index 0000000000000000000000000000000000000000..224d93a7ac1863b745d66a6aa35b91ed11ecd82a
--- /dev/null
+++ b/app/assets/javascripts/streaming/constants.js
@@ -0,0 +1,9 @@
+// Lower min chunk numbers can make the page loading take incredibly long
+export const MIN_CHUNK_SIZE = 128 * 1024;
+export const MAX_CHUNK_SIZE = 2048 * 1024;
+export const LOW_FRAME_TIME = 32;
+// Tasks that take more than 50ms are considered Long
+// https://web.dev/optimize-long-tasks/
+export const HIGH_FRAME_TIME = 64;
+export const BALANCE_RATE = 1.2;
+export const TIMEOUT = 500;
diff --git a/app/assets/javascripts/streaming/handle_streamed_anchor_link.js b/app/assets/javascripts/streaming/handle_streamed_anchor_link.js
new file mode 100644
index 0000000000000000000000000000000000000000..315dc9bb0a0a8974ad410f0458af1ab5ee8c0016
--- /dev/null
+++ b/app/assets/javascripts/streaming/handle_streamed_anchor_link.js
@@ -0,0 +1,26 @@
+import { throttle } from 'lodash';
+import { scrollToElement } from '~/lib/utils/common_utils';
+import LineHighlighter from '~/blob/line_highlighter';
+
+const noop = () => {};
+
+export function handleStreamedAnchorLink(rootElement) {
+ // "#L100-200" → ['L100', 'L200']
+ const [anchorStart, end] = window.location.hash.substring(1).split('-');
+ const anchorEnd = end ? `L${end}` : anchorStart;
+ if (!anchorStart || document.getElementById(anchorEnd)) return noop;
+
+ const handler = throttle((mutationList, instance) => {
+ if (!document.getElementById(anchorEnd)) return;
+ scrollToElement(document.getElementById(anchorStart));
+ // eslint-disable-next-line no-new
+ new LineHighlighter();
+ instance.disconnect();
+ }, 300);
+
+ const observer = new MutationObserver(handler);
+
+ observer.observe(rootElement, { childList: true, subtree: true });
+
+ return () => observer.disconnect();
+}
diff --git a/app/assets/javascripts/streaming/html_stream.js b/app/assets/javascripts/streaming/html_stream.js
new file mode 100644
index 0000000000000000000000000000000000000000..8182f69a60783dba7de0ff7d3f47ea211706bc7c
--- /dev/null
+++ b/app/assets/javascripts/streaming/html_stream.js
@@ -0,0 +1,33 @@
+import { ChunkWriter } from '~/streaming/chunk_writer';
+
+export class HtmlStream {
+ constructor(element) {
+ const streamDocument = document.implementation.createHTMLDocument('stream');
+
+ streamDocument.open();
+ streamDocument.write('<streaming-element>');
+
+ const virtualStreamingElement = streamDocument.querySelector('streaming-element');
+ element.appendChild(document.adoptNode(virtualStreamingElement));
+
+ this.streamDocument = streamDocument;
+ }
+
+ withChunkWriter(config) {
+ return new ChunkWriter(this, config);
+ }
+
+ write(chunk) {
+ // eslint-disable-next-line no-unsanitized/method
+ this.streamDocument.write(chunk);
+ }
+
+ close() {
+ this.streamDocument.write('</streaming-element>');
+ this.streamDocument.close();
+ }
+
+ abort() {
+ this.streamDocument.close();
+ }
+}
diff --git a/app/assets/javascripts/streaming/polyfills.js b/app/assets/javascripts/streaming/polyfills.js
new file mode 100644
index 0000000000000000000000000000000000000000..a9a044a3e99c699ac0efa4d2601831ac6d35f27c
--- /dev/null
+++ b/app/assets/javascripts/streaming/polyfills.js
@@ -0,0 +1,5 @@
+import { createReadableStreamWrapper } from '@mattiasbuelens/web-streams-adapter';
+import { ReadableStream as PolyfillReadableStream } from 'web-streams-polyfill';
+
+// TODO: remove this when our WebStreams API reaches 100% support
+export const toPolyfillReadable = createReadableStreamWrapper(PolyfillReadableStream);
diff --git a/app/assets/javascripts/streaming/rate_limit_stream_requests.js b/app/assets/javascripts/streaming/rate_limit_stream_requests.js
new file mode 100644
index 0000000000000000000000000000000000000000..04a592baa162f20665005eb6850a2cc1cd9dbce0
--- /dev/null
+++ b/app/assets/javascripts/streaming/rate_limit_stream_requests.js
@@ -0,0 +1,87 @@
+const consumeReadableStream = (stream) => {
+ return new Promise((resolve, reject) => {
+ stream.pipeTo(
+ new WritableStream({
+ close: resolve,
+ abort: reject,
+ }),
+ );
+ });
+};
+
+const wait = (timeout) =>
+ new Promise((resolve) => {
+ setTimeout(resolve, timeout);
+ });
+
+// this rate-limiting approach is specific to Web Streams
+// because streams only resolve when they're fully consumed
+// so we need to split each stream into two pieces:
+// one for the rate-limiter (wait for all the bytes to be sent)
+// another for the original consumer
+export const rateLimitStreamRequests = ({
+ factory,
+ total,
+ maxConcurrentRequests,
+ immediateCount = maxConcurrentRequests,
+ timeout = 0,
+}) => {
+ if (total === 0) return [];
+
+ const unsettled = [];
+
+ const pushUnsettled = (promise) => {
+ let res;
+ let rej;
+ const consume = new Promise((resolve, reject) => {
+ res = resolve;
+ rej = reject;
+ });
+ unsettled.push(consume);
+ return promise.then((stream) => {
+ const [first, second] = stream.tee();
+ // eslint-disable-next-line promise/no-nesting
+ consumeReadableStream(first)
+ .then(() => {
+ unsettled.splice(unsettled.indexOf(consume), 1);
+ res();
+ })
+ .catch(rej);
+ return second;
+ }, rej);
+ };
+
+ const immediate = Array.from({ length: Math.min(immediateCount, total) }, (_, i) =>
+ pushUnsettled(factory(i)),
+ );
+
+ const queue = [];
+ const flushQueue = () => {
+ const promises =
+ unsettled.length > maxConcurrentRequests ? unsettled : [...unsettled, wait(timeout)];
+ // errors are handled by the caller
+ // eslint-disable-next-line promise/catch-or-return
+ Promise.race(promises).then(() => {
+ const cb = queue.shift();
+ cb?.();
+ if (queue.length !== 0) {
+ // wait for stream consumer promise to be removed from unsettled
+ queueMicrotask(flushQueue);
+ }
+ });
+ };
+
+ const throttled = Array.from({ length: total - immediateCount }, (_, i) => {
+ return new Promise((resolve, reject) => {
+ queue.push(() => {
+ pushUnsettled(factory(i + immediateCount))
+ .then(resolve)
+ .catch(reject);
+ });
+ });
+ });
+
+ flushQueue();
+
+ return [...immediate, ...throttled];
+};
diff --git a/app/assets/javascripts/streaming/render_balancer.js b/app/assets/javascripts/streaming/render_balancer.js
new file mode 100644
index 0000000000000000000000000000000000000000..66929ff3a54ad422db6e0478cc238f47e5eb7e15
--- /dev/null
+++ b/app/assets/javascripts/streaming/render_balancer.js
@@ -0,0 +1,36 @@
+export class RenderBalancer {
+ previousTimestamp = undefined;
+
+ constructor({ increase, decrease, highFrameTime, lowFrameTime }) {
+ this.increase = increase;
+ this.decrease = decrease;
+ this.highFrameTime = highFrameTime;
+ this.lowFrameTime = lowFrameTime;
+ }
+
+ render(fn) {
+ return new Promise((resolve) => {
+ const callback = (timestamp) => {
+ this.throttle(timestamp);
+ if (fn()) requestAnimationFrame(callback);
+ else resolve();
+ };
+ requestAnimationFrame(callback);
+ });
+ }
+
+ throttle(timestamp) {
+ const { previousTimestamp } = this;
+ this.previousTimestamp = timestamp;
+ if (previousTimestamp === undefined) return;
+
+ const duration = Math.round(timestamp - previousTimestamp);
+ if (!duration) return;
+
+ if (duration >= this.highFrameTime) {
+ this.decrease();
+ } else if (duration < this.lowFrameTime) {
+ this.increase();
+ }
+ }
+}
diff --git a/app/assets/javascripts/streaming/render_html_streams.js b/app/assets/javascripts/streaming/render_html_streams.js
new file mode 100644
index 0000000000000000000000000000000000000000..7201e541777c8d5124200fe25b3bec2b3d8bde67
--- /dev/null
+++ b/app/assets/javascripts/streaming/render_html_streams.js
@@ -0,0 +1,40 @@
+import { HtmlStream } from '~/streaming/html_stream';
+
+async function pipeStreams(domWriter, streamPromises) {
+ try {
+ for await (const stream of streamPromises.slice(0, -1)) {
+ await stream.pipeTo(domWriter, { preventClose: true });
+ }
+ const stream = await streamPromises[streamPromises.length - 1];
+ await stream.pipeTo(domWriter);
+ } catch (error) {
+ domWriter.abort(error);
+ }
+}
+
+// this function (and the rest of the pipeline) expects polyfilled streams
+// do not pass native streams here unless our browser support allows for it
+// TODO: remove this notice when our WebStreams API support reaches 100%
+export function renderHtmlStreams(streamPromises, element, config) {
+ if (streamPromises.length === 0) return Promise.resolve();
+
+ const chunkedHtmlStream = new HtmlStream(element).withChunkWriter(config);
+
+ return new Promise((resolve, reject) => {
+ const domWriter = new WritableStream({
+ write(chunk) {
+ return chunkedHtmlStream.write(chunk);
+ },
+ close() {
+ chunkedHtmlStream.close();
+ resolve();
+ },
+ abort(error) {
+ chunkedHtmlStream.abort();
+ reject(error);
+ },
+ });
+
+ pipeStreams(domWriter, streamPromises);
+ });
+}
diff --git a/app/assets/stylesheets/framework/files.scss b/app/assets/stylesheets/framework/files.scss
index 9ea5a66b3bc4ce96c9b9047fe5d7fe5f2979cb4d..b292adf9eac524ac5de665f94512a7bb9e44719c 100644
--- a/app/assets/stylesheets/framework/files.scss
+++ b/app/assets/stylesheets/framework/files.scss
@@ -580,3 +580,31 @@ span.idiff {
padding: 0;
border-radius: 0 0 $border-radius-default $border-radius-default;
}
+
+.blame-stream-container {
+ border-top: 1px solid $border-color;
+}
+
+.blame-stream-loading {
+ $gradient-size: 16px;
+ position: sticky;
+ bottom: 0;
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ margin-top: -$gradient-size;
+ height: $gl-spacing-scale-10;
+ border-top: $gradient-size solid transparent;
+ background-color: $white;
+ box-sizing: content-box;
+ background-clip: content-box;
+
+ .gradient {
+ position: absolute;
+ left: 0;
+ right: 0;
+ top: -$gradient-size;
+ height: $gradient-size;
+ background: linear-gradient(to top, $white, transparentize($white, 1));
+ }
+}
diff --git a/app/controllers/projects/blame_controller.rb b/app/controllers/projects/blame_controller.rb
index cfff281604e5ddfea85eaaa33b3d188f8ce2a62b..d41b347dc5aa879b8ce926db5dd0d4661bc4b322 100644
--- a/app/controllers/projects/blame_controller.rb
+++ b/app/controllers/projects/blame_controller.rb
@@ -23,13 +23,47 @@ def show
environment_params[:find_latest] = true
@environment = ::Environments::EnvironmentsByDeploymentsFinder.new(@project, current_user, environment_params).execute.last
- blame_service = Projects::BlameService.new(@blob, @commit, params.permit(:page, :no_pagination))
+ permitted_params = params.permit(:page, :no_pagination, :streaming)
+ blame_service = Projects::BlameService.new(@blob, @commit, permitted_params)
@blame = Gitlab::View::Presenter::Factory.new(blame_service.blame, project: @project, path: @path, page: blame_service.page).fabricate!
- @blame_pagination = blame_service.pagination
+ @entire_blame_path = full_blame_path(no_pagination: true)
+ @blame_pages_url = blame_pages_url(permitted_params)
+ if blame_service.streaming_possible
+ @entire_blame_path = full_blame_path(streaming: true)
+ end
+
+ @streaming_enabled = blame_service.streaming_enabled
+ @blame_pagination = blame_service.pagination unless @streaming_enabled
@blame_per_page = blame_service.per_page
+
+ render locals: { total_extra_pages: blame_service.total_extra_pages }
+ end
+
+ def page
+ @blob = @repository.blob_at(@commit.id, @path)
+
+ environment_params = @repository.branch_exists?(@ref) ? { ref: @ref } : { commit: @commit }
+ environment_params[:find_latest] = true
+ @environment = ::Environments::EnvironmentsByDeploymentsFinder.new(@project, current_user, environment_params).execute.last
+
+ blame_service = Projects::BlameService.new(@blob, @commit, params.permit(:page, :streaming))
+
+ @blame = Gitlab::View::Presenter::Factory.new(blame_service.blame, project: @project, path: @path, page: blame_service.page).fabricate!
+
+ render partial: 'page'
+ end
+
+ private
+
+ def full_blame_path(params)
+ namespace_project_blame_path(namespace_id: @project.namespace, project_id: @project, id: @id, **params)
+ end
+
+ def blame_pages_url(params)
+ namespace_project_blame_page_url(namespace_id: @project.namespace, project_id: @project, id: @id, **params)
end
end
diff --git a/app/services/projects/blame_service.rb b/app/services/projects/blame_service.rb
index 58e146e5a32b1026b040d7a15a5dd5d281e9c658..1ea16040655b1cb11288d24ff445e40eaa1a6fd7 100644
--- a/app/services/projects/blame_service.rb
+++ b/app/services/projects/blame_service.rb
@@ -5,15 +5,19 @@
module Projects
class BlameService
PER_PAGE = 1000
+ STREAMING_FIRST_PAGE_SIZE = 200
+ STREAMING_PER_PAGE = 2000
def initialize(blob, commit, params)
@blob = blob
@commit = commit
- @page = extract_page(params)
+ @streaming_enabled = streaming_state(params)
@pagination_enabled = pagination_state(params)
+ @page = extract_page(params)
+ @params = params
end
- attr_reader :page
+ attr_reader :page, :streaming_enabled
def blame
Gitlab::Blame.new(blob, commit, range: blame_range)
@@ -28,7 +32,22 @@ def pagination
end
def per_page
- PER_PAGE
+ streaming_enabled ? STREAMING_PER_PAGE : PER_PAGE
+ end
+
+ def total_pages
+ total = (blob_lines_count.to_f / per_page).ceil
+ return total unless streaming_enabled
+
+ ([blob_lines_count - STREAMING_FIRST_PAGE_SIZE, 0].max.to_f / per_page).ceil + 1
+ end
+
+ def total_extra_pages
+ [total_pages - 1, 0].max
+ end
+
+ def streaming_possible
+ Feature.enabled?(:blame_page_streaming, commit.project)
end
private
@@ -36,9 +55,16 @@ def per_page
attr_reader :blob, :commit, :pagination_enabled
def blame_range
- return unless pagination_enabled
+ return unless pagination_enabled || streaming_enabled
first_line = (page - 1) * per_page + 1
+
+ if streaming_enabled
+ return 1..STREAMING_FIRST_PAGE_SIZE if page == 1
+
+ first_line = STREAMING_FIRST_PAGE_SIZE + (page - 2) * per_page + 1
+ end
+
last_line = (first_line + per_page).to_i - 1
first_line..last_line
@@ -52,6 +78,12 @@ def extract_page(params)
page
end
+ def streaming_state(params)
+ return false unless streaming_possible
+
+ Gitlab::Utils.to_boolean(params[:streaming], default: false)
+ end
+
def pagination_state(params)
return false if Gitlab::Utils.to_boolean(params[:no_pagination], default: false)
@@ -59,7 +91,7 @@ def pagination_state(params)
end
def overlimit?(page)
- page * per_page >= blob_lines_count + per_page
+ page > total_pages
end
def blob_lines_count
diff --git a/app/views/layouts/_head.html.haml b/app/views/layouts/_head.html.haml
index dd441d0d1556b74d8640a06bbb03490479058402..f0c1b0901403ddf40cd86b5c3cf252fd1d3392fd 100644
--- a/app/views/layouts/_head.html.haml
+++ b/app/views/layouts/_head.html.haml
@@ -10,6 +10,7 @@
%meta{ 'http-equiv' => 'X-UA-Compatible', content: 'IE=edge' }
= render 'layouts/startup_js'
+ = yield :startup_js
- if page_canonical_link
%link{ rel: 'canonical', href: page_canonical_link }
diff --git a/app/views/projects/blame/show.html.haml b/app/views/projects/blame/show.html.haml
index 74b85a93c8ee7fcfe9eaf1f77e1a4ce9e1285e69..ee7ca9cd3516667aa32821aa7d4231a152a2a9b4 100644
--- a/app/views/projects/blame/show.html.haml
+++ b/app/views/projects/blame/show.html.haml
@@ -1,6 +1,15 @@
- page_title _("Blame"), @blob.path, @ref
- add_page_specific_style 'page_bundles/tree'
-- dataset = { testid: 'blob-content-holder', qa_selector: 'blame_file_content', per_page: @blame_per_page }
+- if @streaming_enabled && total_extra_pages > 0
+ - content_for :startup_js do
+ = javascript_tag do
+ :plain
+ window.blamePageStream = (() => {
+ const url = new URL("#{@blame_pages_url}");
+ url.searchParams.set('page', 2);
+ return fetch(url).then(response => response.body);
+ })();
+- dataset = { testid: 'blob-content-holder', qa_selector: 'blame_file_content', per_page: @blame_per_page, total_extra_pages: total_extra_pages - 1, pages_url: @blame_pages_url }
#blob-content-holder.tree-holder.js-per-page{ data: dataset }
= render "projects/blob/breadcrumb", blob: @blob, blame: true
@@ -26,11 +35,21 @@
.blame-table-wrapper
= render partial: 'page'
+ - if @streaming_enabled
+ #blame-stream-container.blame-stream-container
+
- if @blame_pagination && @blame_pagination.total_pages > 1
.gl-display-flex.gl-justify-content-center.gl-flex-direction-column.gl-align-items-center.gl-p-3.gl-bg-gray-50.gl-border-t-solid.gl-border-t-1.gl-border-gray-100
- = _('For faster browsing, not all history is shown.')
- = render Pajamas::ButtonComponent.new(href: namespace_project_blame_path(namespace_id: @project.namespace, project_id: @project, id: @id, no_pagination: true), size: :small, button_options: { class: 'gl-mt-3' }) do |c|
- = _('View entire blame')
+ = render Pajamas::ButtonComponent.new(href: @entire_blame_path, size: :small, button_options: { class: 'gl-mt-3' }) do |c|
+ = _('Show full blame')
+
+ - if @streaming_enabled
+ #blame-stream-loading.blame-stream-loading
+ .gradient
+ = gl_loading_icon(size: 'sm')
+ %span.gl-mx-2
+ = _('Loading full blame...')
- if @blame_pagination
= paginate(@blame_pagination, theme: "gitlab")
+
diff --git a/config/feature_flags/development/blame_page_streaming.yml b/config/feature_flags/development/blame_page_streaming.yml
new file mode 100644
index 0000000000000000000000000000000000000000..44d64800dabe62dd0d361cd3ebf045328db02582
--- /dev/null
+++ b/config/feature_flags/development/blame_page_streaming.yml
@@ -0,0 +1,8 @@
+---
+name: blame_page_streaming
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110208
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/392890
+milestone: '15.10'
+type: development
+group: group::source code
+default_enabled: false
diff --git a/config/routes/repository.rb b/config/routes/repository.rb
index 0202eb80b232728ac2869e74852da1b31d1df28f..60d3d37bdc85ae84464693af60c1d8c40cf7f829 100644
--- a/config/routes/repository.rb
+++ b/config/routes/repository.rb
@@ -75,6 +75,7 @@
get '/tree/*id', to: 'tree#show', as: :tree
get '/raw/*id', to: 'raw#show', as: :raw
+ get '/blame_page/*id', to: 'blame#page', as: :blame_page
get '/blame/*id', to: 'blame#show', as: :blame
get '/commits', to: 'commits#commits_root', as: :commits_root
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index a4d914797cf13a5f683f5cd069a413e68aa700a5..2bfb0e533fc2b6ffadce231efeae76f4aa0bd3a2 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -6781,6 +6781,9 @@ msgstr ""
msgid "Blame"
msgstr ""
+msgid "Blame could not be loaded as a single page."
+msgstr ""
+
msgid "BlobViewer|View on %{environmentName}"
msgstr ""
@@ -18006,9 +18009,6 @@ msgstr ""
msgid "For example, the application using the token or the purpose of the token. Do not give sensitive information for the name of the token, as it will be visible to all %{resource_type} members."
msgstr ""
-msgid "For faster browsing, not all history is shown."
-msgstr ""
-
msgid "For files larger than this limit, only index the file name. The file content is neither indexed nor searchable."
msgstr ""
@@ -25708,6 +25708,9 @@ msgstr ""
msgid "Loading files, directories, and submodules in the path %{path} for commit reference %{ref}"
msgstr ""
+msgid "Loading full blame..."
+msgstr ""
+
msgid "Loading more"
msgstr ""
@@ -40219,6 +40222,9 @@ msgstr ""
msgid "Show filters"
msgstr ""
+msgid "Show full blame"
+msgstr ""
+
msgid "Show group milestones"
msgstr ""
@@ -47461,6 +47467,9 @@ msgstr ""
msgid "View blame"
msgstr ""
+msgid "View blame as separate pages"
+msgstr ""
+
msgid "View blame prior to this change"
msgstr ""
@@ -47490,9 +47499,6 @@ msgstr ""
msgid "View eligible approvers"
msgstr ""
-msgid "View entire blame"
-msgstr ""
-
msgid "View exposed artifact"
msgid_plural "View %d exposed artifacts"
msgstr[0] ""
diff --git a/package.json b/package.json
index c5976b96ce374fef70d3fe5457bbe4d327c9fea3..cec754ca1a278818e507d3c59450363b80cb2742 100644
--- a/package.json
+++ b/package.json
@@ -59,6 +59,7 @@
"@gitlab/ui": "56.2.0",
"@gitlab/visual-review-tools": "1.7.3",
"@gitlab/web-ide": "0.0.1-dev-20230223005157",
+ "@mattiasbuelens/web-streams-adapter": "^0.1.0",
"@rails/actioncable": "6.1.4-7",
"@rails/ujs": "6.1.4-7",
"@sourcegraph/code-host-integration": "0.0.84",
@@ -197,6 +198,7 @@
"vue-virtual-scroll-list": "^1.4.7",
"vuedraggable": "^2.23.0",
"vuex": "^3.6.2",
+ "web-streams-polyfill": "^3.2.1",
"web-vitals": "^0.2.4",
"webpack": "^4.46.0",
"webpack-bundle-analyzer": "^4.6.1",
diff --git a/spec/features/projects/blobs/blame_spec.rb b/spec/features/projects/blobs/blame_spec.rb
index 27b7c6ef2d52e95c4aaceb2864281a075a9f681f..d3558af81b8ab899dbead276ca75a2e7ec1e61fd 100644
--- a/spec/features/projects/blobs/blame_spec.rb
+++ b/spec/features/projects/blobs/blame_spec.rb
@@ -38,7 +38,7 @@ def visit_blob_blame(path)
within '[data-testid="blob-content-holder"]' do
expect(page).to have_css('.blame-commit')
expect(page).not_to have_css('.gl-pagination')
- expect(page).not_to have_link _('View entire blame')
+ expect(page).not_to have_link _('Show full blame')
end
end
@@ -53,7 +53,7 @@ def visit_blob_blame(path)
within '[data-testid="blob-content-holder"]' do
expect(page).to have_css('.blame-commit')
expect(page).to have_css('.gl-pagination')
- expect(page).to have_link _('View entire blame')
+ expect(page).to have_link _('Show full blame')
expect(page).to have_css('#L1')
expect(page).not_to have_css('#L3')
@@ -85,19 +85,42 @@ def visit_blob_blame(path)
end
end
- context 'when user clicks on View entire blame button' do
+ shared_examples 'a full blame page' do
+ context 'when user clicks on Show full blame button' do
+ before do
+ visit_blob_blame(path)
+ click_link _('Show full blame')
+ end
+
+ it 'displays the blame page without pagination' do
+ within '[data-testid="blob-content-holder"]' do
+ expect(page).to have_css('#L1')
+ expect(page).to have_css('#L667')
+ expect(page).not_to have_css('.gl-pagination')
+ end
+ end
+ end
+ end
+
+ context 'when streaming is disabled' do
before do
- visit_blob_blame(path)
+ stub_feature_flags(blame_page_streaming: false)
end
- it 'displays the blame page without pagination' do
- within '[data-testid="blob-content-holder"]' do
- click_link _('View entire blame')
+ it_behaves_like 'a full blame page'
+ end
- expect(page).to have_css('#L1')
- expect(page).to have_css('#L3')
- expect(page).not_to have_css('.gl-pagination')
- end
+ context 'when streaming is enabled' do
+ before do
+ stub_const('Projects::BlameService::STREAMING_PER_PAGE', 50)
+ end
+
+ it_behaves_like 'a full blame page'
+
+ it 'shows loading text' do
+ visit_blob_blame(path)
+ click_link _('Show full blame')
+ expect(page).to have_text('Loading full blame...')
end
end
@@ -112,7 +135,7 @@ def visit_blob_blame(path)
within '[data-testid="blob-content-holder"]' do
expect(page).to have_css('.blame-commit')
expect(page).not_to have_css('.gl-pagination')
- expect(page).not_to have_link _('View entire blame')
+ expect(page).not_to have_link _('Show full blame')
end
end
end
diff --git a/spec/frontend/__helpers__/shared_test_setup.js b/spec/frontend/__helpers__/shared_test_setup.js
index 2fe9fe89a90ec2303108dd0c43b317bc25956326..7fc81cf65481e6afdada5c369b0b7ec5b472103c 100644
--- a/spec/frontend/__helpers__/shared_test_setup.js
+++ b/spec/frontend/__helpers__/shared_test_setup.js
@@ -1,4 +1,5 @@
/* Common setup for both unit and integration test environments */
+import { ReadableStream, WritableStream } from 'node:stream/web';
import * as jqueryMatchers from 'custom-jquery-matchers';
import Vue from 'vue';
import { enableAutoDestroy } from '@vue/test-utils';
@@ -13,6 +14,9 @@ import './dom_shims';
import './jquery';
import '~/commons/bootstrap';
+global.ReadableStream = ReadableStream;
+global.WritableStream = WritableStream;
+
enableAutoDestroy(afterEach);
// This module has some fairly decent visual test coverage in it's own repository.
diff --git a/spec/frontend/__mocks__/lodash/debounce.js b/spec/frontend/__mocks__/lodash/debounce.js
index d4fe2ce54063c57ba58595c4cc0e5e3b0c308dd9..15f806fc31a1c05c7a0e0a8692438e1dd644baef 100644
--- a/spec/frontend/__mocks__/lodash/debounce.js
+++ b/spec/frontend/__mocks__/lodash/debounce.js
@@ -9,9 +9,22 @@
// Further reference: https://github.com/facebook/jest/issues/3465
export default (fn) => {
- const debouncedFn = jest.fn().mockImplementation(fn);
- debouncedFn.cancel = jest.fn();
- debouncedFn.flush = jest.fn().mockImplementation(() => {
+ let id;
+ const debouncedFn = jest.fn(function run(...args) {
+ // this is calculated in runtime so beforeAll hook works in tests
+ const timeout = global.JEST_DEBOUNCE_THROTTLE_TIMEOUT;
+ if (timeout) {
+ id = setTimeout(() => {
+ fn.apply(this, args);
+ }, timeout);
+ } else {
+ fn.apply(this, args);
+ }
+ });
+ debouncedFn.cancel = jest.fn(() => {
+ clearTimeout(id);
+ });
+ debouncedFn.flush = jest.fn(() => {
const errorMessage =
"The .flush() method returned by lodash.debounce is not yet implemented/mocked by the mock in 'spec/frontend/__mocks__/lodash/debounce.js'.";
diff --git a/spec/frontend/__mocks__/lodash/throttle.js b/spec/frontend/__mocks__/lodash/throttle.js
index e8a82654c78a70eb41ff1ec9091eeccaa8c39aca..b1014662918029de664d33aeb539ca295715782c 100644
--- a/spec/frontend/__mocks__/lodash/throttle.js
+++ b/spec/frontend/__mocks__/lodash/throttle.js
@@ -1,4 +1,4 @@
// Similar to `lodash/debounce`, `lodash/throttle` also causes flaky specs.
// See `./debounce.js` for more details.
-export default (fn) => fn;
+export { default } from './debounce';
diff --git a/spec/frontend/blame/streaming/index_spec.js b/spec/frontend/blame/streaming/index_spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..a5069f8a7d868bc207238267fff7e6f5402b99db
--- /dev/null
+++ b/spec/frontend/blame/streaming/index_spec.js
@@ -0,0 +1,110 @@
+import waitForPromises from 'helpers/wait_for_promises';
+import { renderBlamePageStreams } from '~/blame/streaming';
+import { setHTMLFixture } from 'helpers/fixtures';
+import { renderHtmlStreams } from '~/streaming/render_html_streams';
+import { rateLimitStreamRequests } from '~/streaming/rate_limit_stream_requests';
+import { handleStreamedAnchorLink } from '~/streaming/handle_streamed_anchor_link';
+import { toPolyfillReadable } from '~/streaming/polyfills';
+import { createAlert } from '~/flash';
+
+jest.mock('~/streaming/render_html_streams');
+jest.mock('~/streaming/rate_limit_stream_requests');
+jest.mock('~/streaming/handle_streamed_anchor_link');
+jest.mock('~/streaming/polyfills');
+jest.mock('~/sentry');
+jest.mock('~/flash');
+
+global.fetch = jest.fn();
+
+describe('renderBlamePageStreams', () => {
+ let stopAnchor;
+ const PAGES_URL = 'https://example.com/';
+ const findStreamContainer = () => document.querySelector('#blame-stream-container');
+ const findStreamLoadingIndicator = () => document.querySelector('#blame-stream-loading');
+
+ const setupHtml = (totalExtraPages = 0) => {
+ setHTMLFixture(`
+ <div id="blob-content-holder"
+ data-total-extra-pages="${totalExtraPages}"
+ data-pages-url="${PAGES_URL}"
+ ></div>
+ <div id="blame-stream-container"></div>
+ <div id="blame-stream-loading"></div>
+ `);
+ };
+
+ handleStreamedAnchorLink.mockImplementation(() => stopAnchor);
+ rateLimitStreamRequests.mockImplementation(({ factory, total }) => {
+ return Array.from({ length: total }, (_, i) => {
+ return Promise.resolve(factory(i));
+ });
+ });
+ toPolyfillReadable.mockImplementation((obj) => obj);
+
+ beforeEach(() => {
+ stopAnchor = jest.fn();
+ fetch.mockClear();
+ });
+
+ it('does nothing for an empty page', async () => {
+ await renderBlamePageStreams();
+
+ expect(handleStreamedAnchorLink).not.toHaveBeenCalled();
+ expect(renderHtmlStreams).not.toHaveBeenCalled();
+ });
+
+ it('renders a single stream', async () => {
+ let res;
+ const stream = new Promise((resolve) => {
+ res = resolve;
+ });
+ renderHtmlStreams.mockImplementationOnce(() => stream);
+ setupHtml();
+
+ renderBlamePageStreams(stream);
+
+ expect(handleStreamedAnchorLink).toHaveBeenCalledTimes(1);
+ expect(stopAnchor).toHaveBeenCalledTimes(0);
+ expect(renderHtmlStreams).toHaveBeenCalledWith([stream], findStreamContainer());
+ expect(findStreamLoadingIndicator()).not.toBe(null);
+
+ res();
+ await waitForPromises();
+
+ expect(stopAnchor).toHaveBeenCalledTimes(1);
+ expect(findStreamLoadingIndicator()).toBe(null);
+ });
+
+ it('renders rest of the streams', async () => {
+ const stream = Promise.resolve();
+ const stream2 = Promise.resolve({ body: null });
+ fetch.mockImplementationOnce(() => stream2);
+ setupHtml(1);
+
+ await renderBlamePageStreams(stream);
+
+ expect(fetch.mock.calls[0][0].toString()).toBe(`${PAGES_URL}?page=3`);
+ expect(renderHtmlStreams).toHaveBeenCalledWith([stream, stream2], findStreamContainer());
+ });
+
+ it('shows an error message when failed', async () => {
+ const stream = Promise.resolve();
+ const error = new Error();
+ renderHtmlStreams.mockImplementationOnce(() => Promise.reject(error));
+ setupHtml();
+
+ try {
+ await renderBlamePageStreams(stream);
+ } catch (err) {
+ expect(err).toBe(error);
+ }
+
+ expect(createAlert).toHaveBeenCalledWith({
+ message: 'Blame could not be loaded as a single page.',
+ primaryButton: {
+ text: 'View blame as separate pages',
+ clickHandler: expect.any(Function),
+ },
+ });
+ });
+});
diff --git a/spec/frontend/streaming/chunk_writer_spec.js b/spec/frontend/streaming/chunk_writer_spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..2aadb3328386778c0b4e942f1e2629ea5e5bd4eb
--- /dev/null
+++ b/spec/frontend/streaming/chunk_writer_spec.js
@@ -0,0 +1,214 @@
+import { ChunkWriter } from '~/streaming/chunk_writer';
+import { RenderBalancer } from '~/streaming/render_balancer';
+
+jest.mock('~/streaming/render_balancer');
+
+describe('ChunkWriter', () => {
+ let accumulator = '';
+ let write;
+ let close;
+ let abort;
+ let config;
+ let render;
+
+ const createChunk = (text) => {
+ const encoder = new TextEncoder();
+ return encoder.encode(text);
+ };
+
+ const createHtmlStream = () => {
+ write = jest.fn((part) => {
+ accumulator += part;
+ });
+ close = jest.fn();
+ abort = jest.fn();
+ return {
+ write,
+ close,
+ abort,
+ };
+ };
+
+ const createWriter = () => {
+ return new ChunkWriter(createHtmlStream(), config);
+ };
+
+ const pushChunks = (...chunks) => {
+ const writer = createWriter();
+ chunks.forEach((chunk) => {
+ writer.write(createChunk(chunk));
+ });
+ writer.close();
+ };
+
+ afterAll(() => {
+ global.JEST_DEBOUNCE_THROTTLE_TIMEOUT = undefined;
+ });
+
+ beforeEach(() => {
+ global.JEST_DEBOUNCE_THROTTLE_TIMEOUT = 100;
+ accumulator = '';
+ config = undefined;
+ render = jest.fn((cb) => {
+ while (cb()) {
+ // render until 'false'
+ }
+ });
+ RenderBalancer.mockImplementation(() => ({ render }));
+ });
+
+ describe('when chunk length must be "1"', () => {
+ beforeEach(() => {
+ config = { minChunkSize: 1, maxChunkSize: 1 };
+ });
+
+ it('splits big chunks into smaller ones', () => {
+ const text = 'foobar';
+ pushChunks(text);
+ expect(accumulator).toBe(text);
+ expect(write).toHaveBeenCalledTimes(text.length);
+ });
+
+ it('handles small emoji chunks', () => {
+ const text = 'foo👀bar👨â€ðŸ‘©â€ðŸ‘§baz👧👧ðŸ»ðŸ‘§ðŸ¼ðŸ‘§ðŸ½ðŸ‘§ðŸ¾ðŸ‘§ðŸ¿';
+ pushChunks(text);
+ expect(accumulator).toBe(text);
+ expect(write).toHaveBeenCalledTimes(createChunk(text).length);
+ });
+ });
+
+ describe('when chunk length must not be lower than "5" and exceed "10"', () => {
+ beforeEach(() => {
+ config = { minChunkSize: 5, maxChunkSize: 10 };
+ });
+
+ it('joins small chunks', () => {
+ const text = '12345';
+ pushChunks(...text.split(''));
+ expect(accumulator).toBe(text);
+ expect(write).toHaveBeenCalledTimes(1);
+ expect(close).toHaveBeenCalledTimes(1);
+ });
+
+ it('handles overflow with small chunks', () => {
+ const text = '123456789';
+ pushChunks(...text.split(''));
+ expect(accumulator).toBe(text);
+ expect(write).toHaveBeenCalledTimes(2);
+ expect(close).toHaveBeenCalledTimes(1);
+ });
+
+ it('calls flush on small chunks', () => {
+ global.JEST_DEBOUNCE_THROTTLE_TIMEOUT = undefined;
+ const flushAccumulator = jest.spyOn(ChunkWriter.prototype, 'flushAccumulator');
+ const text = '1';
+ pushChunks(text);
+ expect(accumulator).toBe(text);
+ expect(flushAccumulator).toHaveBeenCalledTimes(1);
+ });
+
+ it('calls flush on large chunks', () => {
+ const flushAccumulator = jest.spyOn(ChunkWriter.prototype, 'flushAccumulator');
+ const text = '1234567890123';
+ const writer = createWriter();
+ writer.write(createChunk(text));
+ jest.runAllTimers();
+ expect(accumulator).toBe(text);
+ expect(flushAccumulator).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ describe('chunk balancing', () => {
+ let increase;
+ let decrease;
+ let renderOnce;
+
+ beforeEach(() => {
+ render = jest.fn((cb) => {
+ let next = true;
+ renderOnce = () => {
+ if (!next) return;
+ next = cb();
+ };
+ });
+ RenderBalancer.mockImplementation(({ increase: inc, decrease: dec }) => {
+ increase = jest.fn(inc);
+ decrease = jest.fn(dec);
+ return {
+ render,
+ };
+ });
+ });
+
+ describe('when frame time exceeds low limit', () => {
+ beforeEach(() => {
+ config = {
+ minChunkSize: 1,
+ maxChunkSize: 5,
+ balanceRate: 10,
+ };
+ });
+
+ it('increases chunk size', () => {
+ const text = '111222223';
+ const writer = createWriter();
+ const chunk = createChunk(text);
+
+ writer.write(chunk);
+
+ renderOnce();
+ increase();
+ renderOnce();
+ renderOnce();
+
+ writer.close();
+
+ expect(accumulator).toBe(text);
+ expect(write.mock.calls).toMatchObject([['111'], ['22222'], ['3']]);
+ expect(close).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ describe('when frame time exceeds high limit', () => {
+ beforeEach(() => {
+ config = {
+ minChunkSize: 1,
+ maxChunkSize: 10,
+ balanceRate: 2,
+ };
+ });
+
+ it('decreases chunk size', () => {
+ const text = '1111112223345';
+ const writer = createWriter();
+ const chunk = createChunk(text);
+
+ writer.write(chunk);
+
+ renderOnce();
+ decrease();
+
+ renderOnce();
+ decrease();
+
+ renderOnce();
+ decrease();
+
+ renderOnce();
+ renderOnce();
+
+ writer.close();
+
+ expect(accumulator).toBe(text);
+ expect(write.mock.calls).toMatchObject([['111111'], ['222'], ['33'], ['4'], ['5']]);
+ expect(close).toHaveBeenCalledTimes(1);
+ });
+ });
+ });
+
+ it('calls abort on htmlStream', () => {
+ const writer = createWriter();
+ writer.abort();
+ expect(abort).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/spec/frontend/streaming/handle_streamed_anchor_link_spec.js b/spec/frontend/streaming/handle_streamed_anchor_link_spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..ef17957b2fc6a13b342878f8e8917e5696230541
--- /dev/null
+++ b/spec/frontend/streaming/handle_streamed_anchor_link_spec.js
@@ -0,0 +1,132 @@
+import { resetHTMLFixture, setHTMLFixture } from 'helpers/fixtures';
+import waitForPromises from 'helpers/wait_for_promises';
+import { handleStreamedAnchorLink } from '~/streaming/handle_streamed_anchor_link';
+import { scrollToElement } from '~/lib/utils/common_utils';
+import LineHighlighter from '~/blob/line_highlighter';
+import { TEST_HOST } from 'spec/test_constants';
+
+jest.mock('~/lib/utils/common_utils');
+jest.mock('~/blob/line_highlighter');
+
+describe('handleStreamedAnchorLink', () => {
+ const ANCHOR_START = 'L100';
+ const ANCHOR_END = '300';
+ const findRoot = () => document.querySelector('#root');
+
+ afterEach(() => {
+ resetHTMLFixture();
+ });
+
+ describe('when single line anchor is given', () => {
+ beforeEach(() => {
+ delete window.location;
+ window.location = new URL(`${TEST_HOST}#${ANCHOR_START}`);
+ });
+
+ describe('when element is present', () => {
+ beforeEach(() => {
+ setHTMLFixture(`<div id="root"><div id="${ANCHOR_START}"></div></div>`);
+ handleStreamedAnchorLink(findRoot());
+ });
+
+ it('does nothing', async () => {
+ await waitForPromises();
+ expect(scrollToElement).not.toHaveBeenCalled();
+ });
+ });
+
+ describe('when element is streamed', () => {
+ let stop;
+ const insertElement = () => {
+ findRoot().insertAdjacentHTML('afterbegin', `<div id="${ANCHOR_START}"></div>`);
+ };
+
+ beforeEach(() => {
+ setHTMLFixture('<div id="root"></div>');
+ stop = handleStreamedAnchorLink(findRoot());
+ });
+
+ afterEach(() => {
+ stop = undefined;
+ });
+
+ it('scrolls to the anchor when inserted', async () => {
+ insertElement();
+ await waitForPromises();
+ expect(scrollToElement).toHaveBeenCalledTimes(1);
+ expect(LineHighlighter).toHaveBeenCalledTimes(1);
+ });
+
+ it("doesn't scroll to the anchor when destroyed", async () => {
+ stop();
+ insertElement();
+ await waitForPromises();
+ expect(scrollToElement).not.toHaveBeenCalled();
+ });
+ });
+ });
+
+ describe('when line range anchor is given', () => {
+ beforeEach(() => {
+ delete window.location;
+ window.location = new URL(`${TEST_HOST}#${ANCHOR_START}-${ANCHOR_END}`);
+ });
+
+ describe('when last element is present', () => {
+ beforeEach(() => {
+ setHTMLFixture(`<div id="root"><div id="L${ANCHOR_END}"></div></div>`);
+ handleStreamedAnchorLink(findRoot());
+ });
+
+ it('does nothing', async () => {
+ await waitForPromises();
+ expect(scrollToElement).not.toHaveBeenCalled();
+ });
+ });
+
+ describe('when last element is streamed', () => {
+ let stop;
+ const insertElement = () => {
+ findRoot().insertAdjacentHTML(
+ 'afterbegin',
+ `<div id="${ANCHOR_START}"></div><div id="L${ANCHOR_END}"></div>`,
+ );
+ };
+
+ beforeEach(() => {
+ setHTMLFixture('<div id="root"></div>');
+ stop = handleStreamedAnchorLink(findRoot());
+ });
+
+ afterEach(() => {
+ stop = undefined;
+ });
+
+ it('scrolls to the anchor when inserted', async () => {
+ insertElement();
+ await waitForPromises();
+ expect(scrollToElement).toHaveBeenCalledTimes(1);
+ expect(LineHighlighter).toHaveBeenCalledTimes(1);
+ });
+
+ it("doesn't scroll to the anchor when destroyed", async () => {
+ stop();
+ insertElement();
+ await waitForPromises();
+ expect(scrollToElement).not.toHaveBeenCalled();
+ });
+ });
+ });
+
+ describe('when anchor is not given', () => {
+ beforeEach(() => {
+ setHTMLFixture(`<div id="root"></div>`);
+ handleStreamedAnchorLink(findRoot());
+ });
+
+ it('does nothing', async () => {
+ await waitForPromises();
+ expect(scrollToElement).not.toHaveBeenCalled();
+ });
+ });
+});
diff --git a/spec/frontend/streaming/html_stream_spec.js b/spec/frontend/streaming/html_stream_spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..115a9ddc803a2c9e6652edb478e5e25bea07a108
--- /dev/null
+++ b/spec/frontend/streaming/html_stream_spec.js
@@ -0,0 +1,46 @@
+import { HtmlStream } from '~/streaming/html_stream';
+import { ChunkWriter } from '~/streaming/chunk_writer';
+
+jest.mock('~/streaming/chunk_writer');
+
+describe('HtmlStream', () => {
+ let write;
+ let close;
+ let streamingElement;
+
+ beforeEach(() => {
+ write = jest.fn();
+ close = jest.fn();
+ jest.spyOn(Document.prototype, 'write').mockImplementation(write);
+ jest.spyOn(Document.prototype, 'close').mockImplementation(close);
+ jest.spyOn(Document.prototype, 'querySelector').mockImplementation(() => {
+ streamingElement = document.createElement('div');
+ return streamingElement;
+ });
+ });
+
+ it('attaches to original document', () => {
+ // eslint-disable-next-line no-new
+ new HtmlStream(document.body);
+ expect(document.body.contains(streamingElement)).toBe(true);
+ });
+
+ it('can write to a document', () => {
+ const htmlStream = new HtmlStream(document.body);
+ htmlStream.write('foo');
+ htmlStream.close();
+ expect(write.mock.calls).toEqual([['<streaming-element>'], ['foo'], ['</streaming-element>']]);
+ expect(close).toHaveBeenCalledTimes(1);
+ });
+
+ it('returns chunked writer', () => {
+ const htmlStream = new HtmlStream(document.body).withChunkWriter();
+ expect(htmlStream).toBeInstanceOf(ChunkWriter);
+ });
+
+ it('closes on abort', () => {
+ const htmlStream = new HtmlStream(document.body);
+ htmlStream.abort();
+ expect(close).toHaveBeenCalled();
+ });
+});
diff --git a/spec/frontend/streaming/rate_limit_stream_requests_spec.js b/spec/frontend/streaming/rate_limit_stream_requests_spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..02e3cf93014acbb56bd8d142f94425f508f45a9a
--- /dev/null
+++ b/spec/frontend/streaming/rate_limit_stream_requests_spec.js
@@ -0,0 +1,155 @@
+import waitForPromises from 'helpers/wait_for_promises';
+import { rateLimitStreamRequests } from '~/streaming/rate_limit_stream_requests';
+
+describe('rateLimitStreamRequests', () => {
+ const encoder = new TextEncoder('utf-8');
+ const createStreamResponse = (content = 'foo') =>
+ new ReadableStream({
+ pull(controller) {
+ controller.enqueue(encoder.encode(content));
+ controller.close();
+ },
+ });
+
+ const createFactory = (content) => {
+ return jest.fn(() => {
+ return Promise.resolve(createStreamResponse(content));
+ });
+ };
+
+ it('does nothing for zero total requests', () => {
+ const factory = jest.fn();
+ const requests = rateLimitStreamRequests({
+ factory,
+ total: 0,
+ });
+ expect(factory).toHaveBeenCalledTimes(0);
+ expect(requests.length).toBe(0);
+ });
+
+ it('does not exceed total requests', () => {
+ const factory = createFactory();
+ const requests = rateLimitStreamRequests({
+ factory,
+ immediateCount: 100,
+ maxConcurrentRequests: 100,
+ total: 2,
+ });
+ expect(factory).toHaveBeenCalledTimes(2);
+ expect(requests.length).toBe(2);
+ });
+
+ it('creates immediate requests', () => {
+ const factory = createFactory();
+ const requests = rateLimitStreamRequests({
+ factory,
+ maxConcurrentRequests: 2,
+ total: 2,
+ });
+ expect(factory).toHaveBeenCalledTimes(2);
+ expect(requests.length).toBe(2);
+ });
+
+ it('returns correct values', async () => {
+ const fixture = 'foobar';
+ const factory = createFactory(fixture);
+ const requests = rateLimitStreamRequests({
+ factory,
+ maxConcurrentRequests: 2,
+ total: 2,
+ });
+
+ const decoder = new TextDecoder('utf-8');
+ let result = '';
+ for await (const stream of requests) {
+ await stream.pipeTo(
+ new WritableStream({
+ // eslint-disable-next-line no-loop-func
+ write(content) {
+ result += decoder.decode(content);
+ },
+ }),
+ );
+ }
+
+ expect(result).toBe(fixture + fixture);
+ });
+
+ it('delays rate limited requests', async () => {
+ const factory = createFactory();
+ const requests = rateLimitStreamRequests({
+ factory,
+ maxConcurrentRequests: 2,
+ total: 3,
+ });
+ expect(factory).toHaveBeenCalledTimes(2);
+ expect(requests.length).toBe(3);
+
+ await waitForPromises();
+
+ expect(factory).toHaveBeenCalledTimes(3);
+ });
+
+ it('runs next request after previous has been fulfilled', async () => {
+ let res;
+ const factory = jest
+ .fn()
+ .mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ res = resolve;
+ }),
+ )
+ .mockImplementationOnce(() => Promise.resolve(createStreamResponse()));
+ const requests = rateLimitStreamRequests({
+ factory,
+ maxConcurrentRequests: 1,
+ total: 2,
+ });
+ expect(factory).toHaveBeenCalledTimes(1);
+ expect(requests.length).toBe(2);
+
+ await waitForPromises();
+
+ expect(factory).toHaveBeenCalledTimes(1);
+
+ res(createStreamResponse());
+
+ await waitForPromises();
+
+ expect(factory).toHaveBeenCalledTimes(2);
+ });
+
+ it('uses timer to schedule next request', async () => {
+ let res;
+ const factory = jest
+ .fn()
+ .mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ res = resolve;
+ }),
+ )
+ .mockImplementationOnce(() => Promise.resolve(createStreamResponse()));
+ const requests = rateLimitStreamRequests({
+ factory,
+ immediateCount: 1,
+ maxConcurrentRequests: 2,
+ total: 2,
+ timeout: 9999,
+ });
+ expect(factory).toHaveBeenCalledTimes(1);
+ expect(requests.length).toBe(2);
+
+ await waitForPromises();
+
+ expect(factory).toHaveBeenCalledTimes(1);
+
+ jest.runAllTimers();
+
+ await waitForPromises();
+
+ expect(factory).toHaveBeenCalledTimes(2);
+ res(createStreamResponse());
+ });
+});
diff --git a/spec/frontend/streaming/render_balancer_spec.js b/spec/frontend/streaming/render_balancer_spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..dae0c98d678d88ff28547a3ef1fe64c4f65ddfc8
--- /dev/null
+++ b/spec/frontend/streaming/render_balancer_spec.js
@@ -0,0 +1,69 @@
+import { RenderBalancer } from '~/streaming/render_balancer';
+
+const HIGH_FRAME_TIME = 100;
+const LOW_FRAME_TIME = 10;
+
+describe('renderBalancer', () => {
+ let frameTime = 0;
+ let frameTimeDelta = 0;
+ let decrease;
+ let increase;
+
+ const createBalancer = () => {
+ decrease = jest.fn();
+ increase = jest.fn();
+ return new RenderBalancer({
+ highFrameTime: HIGH_FRAME_TIME,
+ lowFrameTime: LOW_FRAME_TIME,
+ increase,
+ decrease,
+ });
+ };
+
+ const renderTimes = (times) => {
+ const balancer = createBalancer();
+ return new Promise((resolve) => {
+ let counter = 0;
+ balancer.render(() => {
+ if (counter === times) {
+ resolve(counter);
+ return false;
+ }
+ counter += 1;
+ return true;
+ });
+ });
+ };
+
+ beforeEach(() => {
+ jest.spyOn(window, 'requestAnimationFrame').mockImplementation((cb) => {
+ frameTime += frameTimeDelta;
+ cb(frameTime);
+ });
+ });
+
+ afterEach(() => {
+ window.requestAnimationFrame.mockRestore();
+ frameTime = 0;
+ frameTimeDelta = 0;
+ });
+
+ it('renders in a loop', async () => {
+ const count = await renderTimes(5);
+ expect(count).toBe(5);
+ });
+
+ it('calls decrease', async () => {
+ frameTimeDelta = 200;
+ await renderTimes(5);
+ expect(decrease).toHaveBeenCalled();
+ expect(increase).not.toHaveBeenCalled();
+ });
+
+ it('calls increase', async () => {
+ frameTimeDelta = 1;
+ await renderTimes(5);
+ expect(increase).toHaveBeenCalled();
+ expect(decrease).not.toHaveBeenCalled();
+ });
+});
diff --git a/spec/frontend/streaming/render_html_streams_spec.js b/spec/frontend/streaming/render_html_streams_spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..55cef0ea4693d49abe992d6bbda3683936720f43
--- /dev/null
+++ b/spec/frontend/streaming/render_html_streams_spec.js
@@ -0,0 +1,96 @@
+import { ReadableStream } from 'node:stream/web';
+import { renderHtmlStreams } from '~/streaming/render_html_streams';
+import { HtmlStream } from '~/streaming/html_stream';
+import waitForPromises from 'helpers/wait_for_promises';
+
+jest.mock('~/streaming/html_stream');
+jest.mock('~/streaming/constants', () => {
+ return {
+ HIGH_FRAME_TIME: 0,
+ LOW_FRAME_TIME: 0,
+ MAX_CHUNK_SIZE: 1,
+ MIN_CHUNK_SIZE: 1,
+ };
+});
+
+const firstStreamContent = 'foobar';
+const secondStreamContent = 'bazqux';
+
+describe('renderHtmlStreams', () => {
+ let htmlWriter;
+ const encoder = new TextEncoder();
+ const createSingleChunkStream = (chunk) => {
+ const encoded = encoder.encode(chunk);
+ const stream = new ReadableStream({
+ pull(controller) {
+ controller.enqueue(encoded);
+ controller.close();
+ },
+ });
+ return [stream, encoded];
+ };
+
+ beforeEach(() => {
+ htmlWriter = {
+ write: jest.fn(),
+ close: jest.fn(),
+ abort: jest.fn(),
+ };
+ jest.spyOn(HtmlStream.prototype, 'withChunkWriter').mockReturnValue(htmlWriter);
+ });
+
+ it('renders a single stream', async () => {
+ const [stream, encoded] = createSingleChunkStream(firstStreamContent);
+
+ await renderHtmlStreams([Promise.resolve(stream)], document.body);
+
+ expect(htmlWriter.write).toHaveBeenCalledWith(encoded);
+ expect(htmlWriter.close).toHaveBeenCalledTimes(1);
+ });
+
+ it('renders stream sequence', async () => {
+ const [stream1, encoded1] = createSingleChunkStream(firstStreamContent);
+ const [stream2, encoded2] = createSingleChunkStream(secondStreamContent);
+
+ await renderHtmlStreams([Promise.resolve(stream1), Promise.resolve(stream2)], document.body);
+
+ expect(htmlWriter.write.mock.calls).toMatchObject([[encoded1], [encoded2]]);
+ expect(htmlWriter.close).toHaveBeenCalledTimes(1);
+ });
+
+ it("doesn't wait for the whole sequence to resolve before streaming", async () => {
+ const [stream1, encoded1] = createSingleChunkStream(firstStreamContent);
+ const [stream2, encoded2] = createSingleChunkStream(secondStreamContent);
+
+ let res;
+ const delayedStream = new Promise((resolve) => {
+ res = resolve;
+ });
+
+ renderHtmlStreams([Promise.resolve(stream1), delayedStream], document.body);
+
+ await waitForPromises();
+
+ expect(htmlWriter.write.mock.calls).toMatchObject([[encoded1]]);
+ expect(htmlWriter.close).toHaveBeenCalledTimes(0);
+
+ res(stream2);
+ await waitForPromises();
+
+ expect(htmlWriter.write.mock.calls).toMatchObject([[encoded1], [encoded2]]);
+ expect(htmlWriter.close).toHaveBeenCalledTimes(1);
+ });
+
+ it('closes HtmlStream on error', async () => {
+ const [stream1] = createSingleChunkStream(firstStreamContent);
+ const error = new Error();
+
+ try {
+ await renderHtmlStreams([Promise.resolve(stream1), Promise.reject(error)], document.body);
+ } catch (err) {
+ expect(err).toBe(error);
+ }
+
+ expect(htmlWriter.abort).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/yarn.lock b/yarn.lock
index 45ddeebeceb00b4a197212065506fa3a2105520e..2b810df506eaa31475d6208aa59fb0433b007478 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1832,6 +1832,11 @@
resolved "https://registry.yarnpkg.com/@linaria/core/-/core-3.0.0-beta.13.tgz#049c5be5faa67e341e413a0f6b641d5d78d91056"
integrity sha512-3zEi5plBCOsEzUneRVuQb+2SAx3qaC1dj0FfFAI6zIJQoDWu0dlSwKijMRack7oO9tUWrchfj3OkKQAd1LBdVg==
+"@mattiasbuelens/web-streams-adapter@^0.1.0":
+ version "0.1.0"
+ resolved "https://registry.yarnpkg.com/@mattiasbuelens/web-streams-adapter/-/web-streams-adapter-0.1.0.tgz#607b5a25682f4ae2741da7ba6df39302505336b3"
+ integrity sha512-oV4PyZfwJNtmFWhvlJLqYIX1Nn22ML8FZpS16ZUKv0hg7414xV1fjsGqxQzLT2dyK92TKxsJSwMOd7VNHAtPmA==
+
"@miragejs/pretender-node-polyfill@^0.1.0":
version "0.1.2"
resolved "https://registry.yarnpkg.com/@miragejs/pretender-node-polyfill/-/pretender-node-polyfill-0.1.2.tgz#d26b6b7483fb70cd62189d05c95d2f67153e43f2"