diff --git a/.gitlab/ci/gitlab-gems.gitlab-ci.yml b/.gitlab/ci/gitlab-gems.gitlab-ci.yml index d5481669076b75aef65574cc1c6eae779cc15455..1ee08c4ab855d2fab3b04b0fbe6954ccca8b5f53 100644 --- a/.gitlab/ci/gitlab-gems.gitlab-ci.yml +++ b/.gitlab/ci/gitlab-gems.gitlab-ci.yml @@ -23,3 +23,6 @@ include: - local: .gitlab/ci/templates/gem.gitlab-ci.yml inputs: gem_name: "gitlab-safe_request_store" + - local: .gitlab/ci/templates/gem.gitlab-ci.yml + inputs: + gem_name: "csv_builder" diff --git a/.rubocop_todo/gitlab/namespaced_class.yml b/.rubocop_todo/gitlab/namespaced_class.yml index 53feb70469806df3a9e89fab2d20efd42e4bda1e..4b000876d0ef69888535759641e12da6209d559b 100644 --- a/.rubocop_todo/gitlab/namespaced_class.yml +++ b/.rubocop_todo/gitlab/namespaced_class.yml @@ -1082,7 +1082,6 @@ Gitlab/NamespacedClass: - 'ee/spec/support/elastic_query_name_inspector.rb' - 'ee/spec/support/test_license.rb' - 'lib/carrier_wave_string_file.rb' - - 'lib/csv_builder.rb' - 'lib/event_filter.rb' - 'lib/file_size_validator.rb' - 'lib/forever.rb' diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml index 797ae5068afc051a3d2c8bf3be1c6717f1ca91dc..2a8e219c754479d189464e67971ed411e0713085 100644 --- a/.rubocop_todo/layout/line_length.yml +++ b/.rubocop_todo/layout/line_length.yml @@ -3697,7 +3697,6 @@ Layout/LineLength: - 'spec/lib/container_registry/client_spec.rb' - 'spec/lib/container_registry/gitlab_api_client_spec.rb' - 'spec/lib/container_registry/registry_spec.rb' - - 'spec/lib/csv_builder_spec.rb' - 'spec/lib/declarative_enum_spec.rb' - 'spec/lib/error_tracking/sentry_client/issue_link_spec.rb' - 'spec/lib/error_tracking/sentry_client/pagination_parser_spec.rb' diff --git a/.rubocop_todo/layout/space_in_lambda_literal.yml b/.rubocop_todo/layout/space_in_lambda_literal.yml index ca6803be7e87501811660a833e2380dee80d98aa..01189989eecb20d6ea812a094559961ecc56c430 100644 --- a/.rubocop_todo/layout/space_in_lambda_literal.yml +++ b/.rubocop_todo/layout/space_in_lambda_literal.yml @@ -371,7 +371,6 @@ Layout/SpaceInLambdaLiteral: - 'spec/helpers/namespaces_helper_spec.rb' - 'spec/lib/backup/gitaly_backup_spec.rb' - 'spec/lib/container_registry/client_spec.rb' - - 'spec/lib/csv_builder_spec.rb' - 'spec/lib/gitlab/analytics/date_filler_spec.rb' - 'spec/lib/gitlab/background_migration/batched_migration_job_spec.rb' - 'spec/lib/gitlab/batch_worker_context_spec.rb' diff --git a/.rubocop_todo/rspec/missing_feature_category.yml b/.rubocop_todo/rspec/missing_feature_category.yml index ba24d335a4097f4d37b0fc959afb3dd23c7b7e8c..a33ab9cf3d486337cc1148b5136df1edb4b896a1 100644 --- a/.rubocop_todo/rspec/missing_feature_category.yml +++ b/.rubocop_todo/rspec/missing_feature_category.yml @@ -2808,8 +2808,6 @@ RSpec/MissingFeatureCategory: - 'spec/lib/container_registry/path_spec.rb' - 'spec/lib/container_registry/registry_spec.rb' - 'spec/lib/container_registry/tag_spec.rb' - - 'spec/lib/csv_builder_spec.rb' - - 'spec/lib/csv_builders/stream_spec.rb' - 'spec/lib/declarative_enum_spec.rb' - 'spec/lib/error_tracking/sentry_client/api_urls_spec.rb' - 'spec/lib/error_tracking/sentry_client/event_spec.rb' diff --git a/.rubocop_todo/rspec/verified_doubles.yml b/.rubocop_todo/rspec/verified_doubles.yml index 0d66bf5604b5fd04d206f95e23f4e91e8b86c616..6f2e884fc55964df96f1f2406def17d45590a852 100644 --- a/.rubocop_todo/rspec/verified_doubles.yml +++ b/.rubocop_todo/rspec/verified_doubles.yml @@ -353,8 +353,6 @@ RSpec/VerifiedDoubles: - 'spec/lib/constraints/jira_encoded_url_constrainer_spec.rb' - 'spec/lib/constraints/project_url_constrainer_spec.rb' - 'spec/lib/constraints/user_url_constrainer_spec.rb' - - 'spec/lib/csv_builder_spec.rb' - - 'spec/lib/csv_builders/stream_spec.rb' - 'spec/lib/extracts_path_spec.rb' - 'spec/lib/feature_spec.rb' - 'spec/lib/gitaly/server_spec.rb' diff --git a/.rubocop_todo/style/redundant_freeze.yml b/.rubocop_todo/style/redundant_freeze.yml index 37269581002a698e9221ff7d97a69266a60bf47d..02eb5c0e64feb4b1bf56aacb741af17df167e950 100644 --- a/.rubocop_todo/style/redundant_freeze.yml +++ b/.rubocop_todo/style/redundant_freeze.yml @@ -79,7 +79,6 @@ Style/RedundantFreeze: - 'lib/banzai/filter/task_list_filter.rb' - 'lib/bulk_imports/common/pipelines/uploads_pipeline.rb' - 'lib/bulk_imports/file_downloads/filename_fetch.rb' - - 'lib/csv_builder.rb' - 'lib/error_tracking/sentry_client/pagination_parser.rb' - 'lib/expand_variables.rb' - 'lib/feature/definition.rb' diff --git a/.rubocop_todo/style/redundant_regexp_escape.yml b/.rubocop_todo/style/redundant_regexp_escape.yml index 7d10d3e20b3a310b039d7113a3d13c781875853f..93aab69237983b1363a3e96773535158f5218b44 100644 --- a/.rubocop_todo/style/redundant_regexp_escape.yml +++ b/.rubocop_todo/style/redundant_regexp_escape.yml @@ -48,7 +48,6 @@ Style/RedundantRegexpEscape: - 'lib/banzai/filter/inline_diff_filter.rb' - 'lib/banzai/filter/task_list_filter.rb' - 'lib/bulk_imports/common/pipelines/uploads_pipeline.rb' - - 'lib/csv_builder.rb' - 'lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb' - 'lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb' - 'lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb' diff --git a/Gemfile b/Gemfile index 64c6d874fe838caece74d6a188deb29608707cf7..34089a5a2f04a85c4e1cd19d9773445c8c0a27f6 100644 --- a/Gemfile +++ b/Gemfile @@ -257,6 +257,7 @@ gem 're2', '~> 1.7.0' gem 'semver_dialects', '~> 1.2.1' gem 'version_sorter', '~> 2.3' +gem 'csv_builder', path: 'gems/csv_builder' # Export Ruby Regex to Javascript gem 'js_regex', '~> 3.8' diff --git a/Gemfile.lock b/Gemfile.lock index cdb0230842ec84cc257ae015f52b56c6425692a6..5c458e397a8d728e7a37e883d5aefd786f828892 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -18,6 +18,11 @@ PATH error_tracking_open_api (1.0.0) typhoeus (~> 1.0, >= 1.0.1) +PATH + remote: gems/csv_builder + specs: + csv_builder (0.1.0) + PATH remote: gems/gitlab-rspec specs: @@ -1760,6 +1765,7 @@ DEPENDENCIES countries (~> 4.0.0) creole (~> 0.5.0) crystalball (~> 0.7.0) + csv_builder! cvss-suite (~> 3.0.1) database_cleaner (~> 1.7.0) deckar01-task_list (= 2.3.2) diff --git a/app/controllers/projects/ci/daily_build_group_report_results_controller.rb b/app/controllers/projects/ci/daily_build_group_report_results_controller.rb index 37138afc7195be9cc7888f9c8cdb64933c1395fa..c1d325d89989501788de0a4a412f60e29fcaafc8 100644 --- a/app/controllers/projects/ci/daily_build_group_report_results_controller.rb +++ b/app/controllers/projects/ci/daily_build_group_report_results_controller.rb @@ -20,7 +20,7 @@ def validate_param_type! end def render_csv(collection) - CsvBuilders::SingleBatch.new( + CsvBuilder::SingleBatch.new( collection, { date: 'date', diff --git a/ee/app/controllers/groups/analytics/coverage_reports_controller.rb b/ee/app/controllers/groups/analytics/coverage_reports_controller.rb index e87204cebff2d679d976997c2c4a3395844f613e..2306b99acd54ab792c1192bb80c3e1ee7deccf75 100644 --- a/ee/app/controllers/groups/analytics/coverage_reports_controller.rb +++ b/ee/app/controllers/groups/analytics/coverage_reports_controller.rb @@ -19,7 +19,7 @@ def index private def render_csv(collection) - CsvBuilders::SingleBatch.new( + CsvBuilder::SingleBatch.new( collection, { date: 'date', diff --git a/ee/app/controllers/projects/security/scanned_resources_controller.rb b/ee/app/controllers/projects/security/scanned_resources_controller.rb index 154d01afc521f6cb93618cf40f8a989ba8c2d562..db92367ba7fca04dba5ba45666c13d6c21881db4 100644 --- a/ee/app/controllers/projects/security/scanned_resources_controller.rb +++ b/ee/app/controllers/projects/security/scanned_resources_controller.rb @@ -34,7 +34,7 @@ def scanned_resources end def render_csv - CsvBuilders::SingleBatch.new( + CsvBuilder::SingleBatch.new( @scanned_resources, { 'Method': 'request_method', diff --git a/ee/app/services/audit_events/export_csv_service.rb b/ee/app/services/audit_events/export_csv_service.rb index 7920bb02ebd97c92c53b1124d7d77a297a008565..e939752f5166cf884d502adb1199d94095dc56d8 100644 --- a/ee/app/services/audit_events/export_csv_service.rb +++ b/ee/app/services/audit_events/export_csv_service.rb @@ -13,7 +13,7 @@ def csv_data private def csv_builder - @csv_builder ||= CsvBuilders::Stream.new(data, header_to_value_hash) + @csv_builder ||= CsvBuilder::Stream.new(data, header_to_value_hash) end def data diff --git a/ee/app/services/groups/seat_usage_export_service.rb b/ee/app/services/groups/seat_usage_export_service.rb index c310627812c1af58f1211d0c0fec009897b3f27f..18bb041fac250edfbed9af01ddac89aa6a6a83bc 100644 --- a/ee/app/services/groups/seat_usage_export_service.rb +++ b/ee/app/services/groups/seat_usage_export_service.rb @@ -31,7 +31,7 @@ def insufficient_permissions end def csv_builder - @csv_builder = CsvBuilders::Stream.new(data, header_to_value_hash) + @csv_builder = CsvBuilder::Stream.new(data, header_to_value_hash) end def data diff --git a/ee/app/services/user_permissions/export_service.rb b/ee/app/services/user_permissions/export_service.rb index de8542a2329031e67a7b2abc4438b2f631891e27..d0f912af9794849398fdc48020f3647cc7374878 100644 --- a/ee/app/services/user_permissions/export_service.rb +++ b/ee/app/services/user_permissions/export_service.rb @@ -21,7 +21,7 @@ def allowed? end def csv_builder - @csv_builder ||= CsvBuilders::Stream.new(data, header_to_value_hash) + @csv_builder ||= CsvBuilder::Stream.new(data, header_to_value_hash) end def data diff --git a/gems/csv_builder/.gitignore b/gems/csv_builder/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b04a8c840df1a534cfd67449e31919721b410986 --- /dev/null +++ b/gems/csv_builder/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/gems/csv_builder/.gitlab-ci.yml b/gems/csv_builder/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..e620c7bacd5474cf31a0a21b49e5efca14513e4f --- /dev/null +++ b/gems/csv_builder/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "csv_builder" \ No newline at end of file diff --git a/gems/csv_builder/.rspec b/gems/csv_builder/.rspec new file mode 100644 index 0000000000000000000000000000000000000000..34c5164d9b56c7d528f061c97f2d2fe02c834bdd --- /dev/null +++ b/gems/csv_builder/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/gems/csv_builder/.rubocop.yml b/gems/csv_builder/.rubocop.yml new file mode 100644 index 0000000000000000000000000000000000000000..d004dd48db72afc988c6acefaac3c2f6c630141a --- /dev/null +++ b/gems/csv_builder/.rubocop.yml @@ -0,0 +1,8 @@ +inherit_from: + - ../config/rubocop.yml + +RSpec/MultipleMemoizedHelpers: + Max: 25 + +RSpec/VerifiedDoubles: + Enabled: false \ No newline at end of file diff --git a/gems/csv_builder/Gemfile b/gems/csv_builder/Gemfile new file mode 100644 index 0000000000000000000000000000000000000000..81bdfcabdae284c2de194d082d548d631b476bfc --- /dev/null +++ b/gems/csv_builder/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in csv_builder.gemspec +gemspec diff --git a/gems/csv_builder/Gemfile.lock b/gems/csv_builder/Gemfile.lock new file mode 100644 index 0000000000000000000000000000000000000000..04992abc4d60a97e796ac0f9ed497ddaf4dddd7b --- /dev/null +++ b/gems/csv_builder/Gemfile.lock @@ -0,0 +1,100 @@ +PATH + remote: . + specs: + csv_builder (0.1.0) + +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + ast (2.4.2) + coderay (1.1.3) + concurrent-ruby (1.2.2) + diff-lcs (1.5.0) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.6.3) + method_source (1.0.0) + minitest (5.19.0) + parallel (1.23.0) + parser (3.2.2.3) + ast (~> 2.4.1) + racc + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.7.1) + rack (3.0.8) + rainbow (3.1.1) + regexp_parser (2.8.1) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.29.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.18.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.23.1) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.18.0) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.20.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rspec (2.22.0) + rubocop (~> 1.33) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.4.2) + +PLATFORMS + ruby + +DEPENDENCIES + csv_builder! + gitlab-styles (~> 10.1.0) + pry + rspec (~> 3.0) + rubocop (~> 1.50) + rubocop-rspec (~> 2.22) + +BUNDLED WITH + 2.4.17 diff --git a/gems/csv_builder/README.md b/gems/csv_builder/README.md new file mode 100644 index 0000000000000000000000000000000000000000..37dde4b334c89c0029f10d4ef5b11449d1005aa9 --- /dev/null +++ b/gems/csv_builder/README.md @@ -0,0 +1,48 @@ +# CsvBuilder + +## Usage + +Generate a CSV given a collection and a mapping. + +```ruby +columns = { + 'Title' => 'title', + 'Comment' => 'comment', + 'Author' => -> (post) { post.author.full_name } + 'Created At (UTC)' => -> (post) { post.created_at&.strftime('%Y-%m-%d %H:%M:%S') } +} + +CsvBuilder.new(@posts, columns).render +``` + +When the value of the mapping is a string, a method is called with the given name +on the record (for example: `post.title`). +When the value of the mapping is a lambda, it is lazily executed. + +It's possible to also pass ActiveRecord associations to preload when batching +through the collection: + +```ruby +CsvBuilder.new(@posts, columns, [:author, :comments]).render +``` + +### SingleBatch builder + +When the collection is an array or enumerable you can use: + +```ruby +CsvBuilder::SingleBatch.new(@posts, columns).render +``` + +### Stream builder + +A stream builder uses a lazy and more efficient iterator and by default returns +up to 100,000 records from the collection. + +```ruby +CsvBuilder::Stream.new(@posts, columns).render(1_000) +``` + +## Development + +Follow the GitLab [gems development guidelines](../../doc/development/gems.md). diff --git a/gems/csv_builder/csv_builder.gemspec b/gems/csv_builder/csv_builder.gemspec new file mode 100644 index 0000000000000000000000000000000000000000..956fe2d610823962e568595be7f75373fa3d3e40 --- /dev/null +++ b/gems/csv_builder/csv_builder.gemspec @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require_relative "lib/csv_builder/version" + +Gem::Specification.new do |spec| + spec.name = "csv_builder" + spec.version = CsvBuilder::Version::VERSION + spec.authors = ["group::tenant-scale"] + spec.email = ["engineering@gitlab.com"] + + spec.summary = "Provides enhancements to the CSV standard library" + spec.description = "Provides enhancements to the CSV standard library" + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/csv_builder" + spec.license = 'MIT' + spec.required_ruby_version = ">= 3.0.0" + spec.metadata["rubygems_mfa_required"] = "true" + + spec.files = Dir['lib/**/*.rb'] + spec.require_paths = ["lib"] + + spec.add_development_dependency "gitlab-styles", "~> 10.1.0" + spec.add_development_dependency "pry" + spec.add_development_dependency "rspec", "~> 3.0" + spec.add_development_dependency "rubocop", "~> 1.50" + spec.add_development_dependency "rubocop-rspec", "~> 2.22" +end diff --git a/gems/csv_builder/lib/csv_builder.rb b/gems/csv_builder/lib/csv_builder.rb new file mode 100644 index 0000000000000000000000000000000000000000..1ef38a1d6a4db2dd755c1096bb1c2859e15e47e5 --- /dev/null +++ b/gems/csv_builder/lib/csv_builder.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'csv' +require 'tempfile' + +require_relative "csv_builder/version" +require_relative "csv_builder/builder" +require_relative "csv_builder/single_batch" +require_relative "csv_builder/stream" + +# Generates CSV when given a collection and a mapping. +# +# Example: +# +# columns = { +# 'Title' => 'title', +# 'Comment' => 'comment', +# 'Author' => -> (post) { post.author.full_name } +# 'Created At (UTC)' => -> (post) { post.created_at&.strftime('%Y-%m-%d %H:%M:%S') } +# } +# +# CsvBuilder.new(@posts, columns).render +# +module CsvBuilder + # + # * +collection+ - The data collection to be used + # * +header_to_value_hash+ - A hash of 'Column Heading' => 'value_method'. + # * +associations_to_preload+ - An array of records to preload with a batch of records. + # + # The value method will be called once for each object in the collection, to + # determine the value for that row. It can either be the name of a method on + # the object, or a lamda to call passing in the object. + def self.new(collection, header_to_value_hash, associations_to_preload = []) + CsvBuilder::Builder.new(collection, header_to_value_hash, associations_to_preload) + end +end diff --git a/gems/csv_builder/lib/csv_builder/builder.rb b/gems/csv_builder/lib/csv_builder/builder.rb new file mode 100644 index 0000000000000000000000000000000000000000..3baa2155fc941723c2d293685c7bcefd02d80da0 --- /dev/null +++ b/gems/csv_builder/lib/csv_builder/builder.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module CsvBuilder + class Builder + UNSAFE_EXCEL_PREFIX = /\A[=\+\-@;]/ # rubocop:disable Style/RedundantRegexpEscape + + attr_reader :rows_written + + def initialize(collection, header_to_value_hash, associations_to_preload = []) + @header_to_value_hash = header_to_value_hash + @collection = collection + @truncated = false + @rows_written = 0 + @associations_to_preload = associations_to_preload + end + + # Renders the csv to a string + def render(truncate_after_bytes = nil) + Tempfile.open(['csv']) do |tempfile| + csv = CSV.new(tempfile) + + write_csv csv, until_condition: -> do + truncate_after_bytes && tempfile.size > truncate_after_bytes + end + + if block_given? + yield tempfile + else + tempfile.rewind + tempfile.read + end + end + end + + def truncated? + @truncated + end + + def rows_expected + if truncated? || rows_written.zero? + @collection.count + else + rows_written + end + end + + def status + { + truncated: truncated?, + rows_written: rows_written, + rows_expected: rows_expected + } + end + + protected + + def each(&block) + if @associations_to_preload&.any? && @collection.respond_to?(:each_batch) + @collection.each_batch(order_hint: :created_at) do |relation| + relation.preload(@associations_to_preload).order(:id).each(&block) + end + else + @collection.find_each(&block) + end + end + + private + + def headers + @headers ||= @header_to_value_hash.keys + end + + def attributes + @attributes ||= @header_to_value_hash.values + end + + def row(object) + attributes.map do |attribute| + if attribute.respond_to?(:call) + excel_sanitize(attribute.call(object)) + else + excel_sanitize(object.public_send(attribute)) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + def write_csv(csv, until_condition:) + csv << headers + + each do |object| + csv << row(object) + + @rows_written += 1 + + if until_condition.call + @truncated = true + break + end + end + end + + def excel_sanitize(line) + return if line.nil? + return line unless line.is_a?(String) && line.match?(UNSAFE_EXCEL_PREFIX) + + ["'", line].join + end + end +end diff --git a/lib/csv_builders/single_batch.rb b/gems/csv_builder/lib/csv_builder/single_batch.rb similarity index 65% rename from lib/csv_builders/single_batch.rb rename to gems/csv_builder/lib/csv_builder/single_batch.rb index bed6b7424b3a6bf8acec95d28b293eee15189b78..e7731f27fd00d91dc90a9a6e6febcfa977c5ad21 100644 --- a/lib/csv_builders/single_batch.rb +++ b/gems/csv_builder/lib/csv_builder/single_batch.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module CsvBuilders - class SingleBatch < CsvBuilder +module CsvBuilder + class SingleBatch < CsvBuilder::Builder protected def each(&block) diff --git a/lib/csv_builders/stream.rb b/gems/csv_builder/lib/csv_builder/stream.rb similarity index 68% rename from lib/csv_builders/stream.rb rename to gems/csv_builder/lib/csv_builder/stream.rb index a2b9fca84cb4bed3450edef503397ae5917116fd..3e1a6c84ce959e90e427f38bf242794b6179621b 100644 --- a/lib/csv_builders/stream.rb +++ b/gems/csv_builder/lib/csv_builder/stream.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module CsvBuilders - class Stream < CsvBuilder +module CsvBuilder + class Stream < CsvBuilder::Builder def render(max_rows = 100_000) max_rows_including_header = max_rows + 1 @@ -11,7 +11,7 @@ def render(max_rows = 100_000) each do |object| csv << CSV.generate_line(row(object)) end - end.lazy.take(max_rows_including_header) # rubocop: disable CodeReuse/ActiveRecord + end.lazy.take(max_rows_including_header) end end end diff --git a/gems/csv_builder/lib/csv_builder/version.rb b/gems/csv_builder/lib/csv_builder/version.rb new file mode 100644 index 0000000000000000000000000000000000000000..b7baf16ad0a0857ce661b4eca50a80054b530165 --- /dev/null +++ b/gems/csv_builder/lib/csv_builder/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module CsvBuilder + module Version + VERSION = "0.1.0" + end +end diff --git a/spec/lib/csv_builders/stream_spec.rb b/gems/csv_builder/spec/csv_builder/stream_spec.rb similarity index 80% rename from spec/lib/csv_builders/stream_spec.rb rename to gems/csv_builder/spec/csv_builder/stream_spec.rb index 7df55fe4230b9e5482ae890aa940fbdd90e0f107..d23e63520afff39323fe44ed169fabc0ca88d4ed 100644 --- a/spec/lib/csv_builders/stream_spec.rb +++ b/gems/csv_builder/spec/csv_builder/stream_spec.rb @@ -2,18 +2,18 @@ require 'spec_helper' -RSpec.describe CsvBuilders::Stream do +RSpec.describe CsvBuilder::Stream do let(:event_1) { double(title: 'Added salt', description: 'A teaspoon') } let(:event_2) { double(title: 'Added sugar', description: 'Just a pinch') } - let(:fake_relation) { FakeRelation.new([event_1, event_2]) } + let(:fake_relation) { described_class::FakeRelation.new([event_1, event_2]) } subject(:builder) { described_class.new(fake_relation, 'Title' => 'title', 'Description' => 'description') } describe '#render' do before do - stub_const('FakeRelation', Array) + stub_const("#{described_class}::FakeRelation", Array) - FakeRelation.class_eval do + described_class::FakeRelation.class_eval do def find_each(&block) each(&block) end diff --git a/spec/lib/csv_builder_spec.rb b/gems/csv_builder/spec/csv_builder_spec.rb similarity index 83% rename from spec/lib/csv_builder_spec.rb rename to gems/csv_builder/spec/csv_builder_spec.rb index ec065ee6f7da44eb030deeedcef384999b632b78..9391938f59d93df9432e56d87b8925f7f7fb82b0 100644 --- a/spec/lib/csv_builder_spec.rb +++ b/gems/csv_builder/spec/csv_builder_spec.rb @@ -1,23 +1,29 @@ # frozen_string_literal: true -require 'spec_helper' - RSpec.describe CsvBuilder do let(:object) { double(question: :answer) } - let(:fake_relation) { FakeRelation.new([object]) } - let(:subject) { described_class.new(fake_relation, 'Q & A' => :question, 'Reversed' => -> (o) { o.question.to_s.reverse }) } + let(:fake_relation) { described_class::FakeRelation.new([object]) } let(:csv_data) { subject.render } + let(:subject) do + described_class.new( + fake_relation, 'Q & A' => :question, 'Reversed' => ->(o) { o.question.to_s.reverse }) + end + before do - stub_const('FakeRelation', Array) + stub_const("#{described_class}::FakeRelation", Array) - FakeRelation.class_eval do + described_class::FakeRelation.class_eval do def find_each(&block) each(&block) end end end + it "has a version number" do + expect(CsvBuilder::Version::VERSION).not_to be nil + end + it 'generates a csv' do expect(csv_data.scan(/(,|\n)/).join).to include ",\n," end @@ -50,7 +56,7 @@ def find_each(&block) describe 'truncation' do let(:big_object) { double(question: 'Long' * 1024) } let(:row_size) { big_object.question.length * 2 } - let(:fake_relation) { FakeRelation.new([big_object, big_object, big_object]) } + let(:fake_relation) { described_class::FakeRelation.new([big_object, big_object, big_object]) } it 'occurs after given number of bytes' do expect(subject.render(row_size * 2).length).to be_between(row_size * 2, row_size * 3) @@ -92,7 +98,7 @@ def find_each(&block) describe 'excel sanitization' do let(:dangerous_title) { double(title: "=cmd|' /C calc'!A0 title", description: "*safe_desc") } let(:dangerous_desc) { double(title: "*safe_title", description: "=cmd|' /C calc'!A0 desc") } - let(:fake_relation) { FakeRelation.new([dangerous_title, dangerous_desc]) } + let(:fake_relation) { described_class::FakeRelation.new([dangerous_title, dangerous_desc]) } let(:subject) { described_class.new(fake_relation, 'Title' => 'title', 'Description' => 'description') } let(:csv_data) { subject.render } @@ -109,7 +115,7 @@ def find_each(&block) context 'when dangerous characters are after a line break' do it 'does not append single quote to description' do fake_object = double(title: "Safe title", description: "With task list\n-[x] todo 1") - fake_relation = FakeRelation.new([fake_object]) + fake_relation = described_class::FakeRelation.new([fake_object]) builder = described_class.new(fake_relation, 'Title' => 'title', 'Description' => 'description') csv_data = builder.render diff --git a/gems/csv_builder/spec/spec_helper.rb b/gems/csv_builder/spec/spec_helper.rb new file mode 100644 index 0000000000000000000000000000000000000000..eb21c2271f625550119085329bc3744882219756 --- /dev/null +++ b/gems/csv_builder/spec/spec_helper.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require "csv_builder" + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/lib/csv_builder.rb b/lib/csv_builder.rb deleted file mode 100644 index a54c355396dcb10049c48ea23acc2ef9d59d606d..0000000000000000000000000000000000000000 --- a/lib/csv_builder.rb +++ /dev/null @@ -1,130 +0,0 @@ -# frozen_string_literal: true - -# Generates CSV when given a collection and a mapping. -# -# Example: -# -# columns = { -# 'Title' => 'title', -# 'Comment' => 'comment', -# 'Author' => -> (post) { post.author.full_name } -# 'Created At (UTC)' => -> (post) { post.created_at&.strftime('%Y-%m-%d %H:%M:%S') } -# } -# -# CsvBuilder.new(@posts, columns).render -# -class CsvBuilder - DEFAULT_ORDER_BY = 'id' - DEFAULT_BATCH_SIZE = 1000 - PREFIX_REGEX = /\A[=\+\-@;]/.freeze - - attr_reader :rows_written - - # - # * +collection+ - The data collection to be used - # * +header_to_hash_value+ - A hash of 'Column Heading' => 'value_method'. - # * +associations_to_preload+ - An array of records to preload with a batch of records. - # - # The value method will be called once for each object in the collection, to - # determine the value for that row. It can either be the name of a method on - # the object, or a lamda to call passing in the object. - def initialize(collection, header_to_value_hash, associations_to_preload = []) - @header_to_value_hash = header_to_value_hash - @collection = collection - @truncated = false - @rows_written = 0 - @associations_to_preload = associations_to_preload - end - - # Renders the csv to a string - def render(truncate_after_bytes = nil) - Tempfile.open(['csv']) do |tempfile| - csv = CSV.new(tempfile) - - write_csv csv, until_condition: -> do - truncate_after_bytes && tempfile.size > truncate_after_bytes - end - - if block_given? - yield tempfile - else - tempfile.rewind - tempfile.read - end - end - end - - def truncated? - @truncated - end - - def rows_expected - if truncated? || rows_written == 0 - @collection.count - else - rows_written - end - end - - def status - { - truncated: truncated?, - rows_written: rows_written, - rows_expected: rows_expected - } - end - - protected - - def each(&block) - if @associations_to_preload.present? && @collection.respond_to?(:each_batch) - @collection.each_batch(order_hint: :created_at) do |relation| - relation.preload(@associations_to_preload).order(:id).each(&block) # rubocop:disable CodeReuse/ActiveRecord - end - else - @collection.find_each(&block) # rubocop: disable CodeReuse/ActiveRecord - end - end - - private - - def headers - @headers ||= @header_to_value_hash.keys - end - - def attributes - @attributes ||= @header_to_value_hash.values - end - - def row(object) - attributes.map do |attribute| - if attribute.respond_to?(:call) - excel_sanitize(attribute.call(object)) - else - excel_sanitize(object.public_send(attribute)) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - - def write_csv(csv, until_condition:) - csv << headers - - each do |object| - csv << row(object) - - @rows_written += 1 - - if until_condition.call - @truncated = true - break - end - end - end - - def excel_sanitize(line) - return if line.nil? - return line unless line.is_a?(String) && line.match?(PREFIX_REGEX) - - ["'", line].join - end -end diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml index 03bf9b5263652289e9cb084e8b7fedddc245da27..55d2b486a6f02a99c06e52225a9020b6a9348ebb 100644 --- a/spec/support/rspec_order_todo.yml +++ b/spec/support/rspec_order_todo.yml @@ -5451,8 +5451,6 @@ - './spec/lib/container_registry/path_spec.rb' - './spec/lib/container_registry/registry_spec.rb' - './spec/lib/container_registry/tag_spec.rb' -- './spec/lib/csv_builder_spec.rb' -- './spec/lib/csv_builders/stream_spec.rb' - './spec/lib/declarative_enum_spec.rb' - './spec/lib/error_tracking/stacktrace_builder_spec.rb' - './spec/lib/event_filter_spec.rb'