From b6ba364f4c4cec72eaa5da11cf89f9de1d6215e5 Mon Sep 17 00:00:00 2001 From: Tomek Date: Thu, 31 Aug 2017 10:26:47 +0200 Subject: [PATCH 1/2] Add pipeline parameter builder --- examples/interpreter/to_sql.rb | 1 + lib/filterly.rb | 1 + lib/filterly/node_builder.rb | 9 +- lib/filterly/pipeline/param_builder.rb | 54 ++++++++ lib/filterly/tree.rb | 30 +++-- spec/examples/interpreter_spec.rb | 2 +- .../to_sql/expression_factory_spec.rb | 2 +- spec/spec_helper.rb | 10 +- spec/unit/node_builder_spec.rb | 25 ++-- spec/unit/parser_spec.rb | 12 +- spec/unit/pipeline/param_builder_spec.rb | 115 ++++++++++++++++++ spec/unit/tree_spec.rb | 15 +++ 12 files changed, 237 insertions(+), 39 deletions(-) create mode 100644 lib/filterly/pipeline/param_builder.rb create mode 100644 spec/unit/pipeline/param_builder_spec.rb diff --git a/examples/interpreter/to_sql.rb b/examples/interpreter/to_sql.rb index 59af090..e10b91a 100644 --- a/examples/interpreter/to_sql.rb +++ b/examples/interpreter/to_sql.rb @@ -29,6 +29,7 @@ def visit_root(_value, left, _right) # @api private def visit_statement(value, left, right) + return visit(left) if right.nil? case value when :and "#{visit(left)} AND #{visit(right)}" diff --git a/lib/filterly.rb b/lib/filterly.rb index 986eac1..e0e1649 100644 --- a/lib/filterly.rb +++ b/lib/filterly.rb @@ -3,6 +3,7 @@ require 'filterly/parser' require 'filterly/tree' require 'filterly/node_builder' +require 'filterly/pipeline/param_builder' module Filterly # Your code goes here... diff --git a/lib/filterly/node_builder.rb b/lib/filterly/node_builder.rb index 28fa922..7d604a9 100644 --- a/lib/filterly/node_builder.rb +++ b/lib/filterly/node_builder.rb @@ -79,18 +79,21 @@ def self.build_custom_node(type:, args:) # @api private def self.attr_array(array_of_values) - Filterly::Node.new(:attr_array, [nil, ast_array(array_of_values), nil]) + Filterly::Node.new( + :attr_array, + attr_array_params(array_of_values) + ) end # @api private def self.ast_array(params) return if params.nil? - return Filterly::Node.new(:attr_value, [params, nil, nil]) unless + return Filterly::Node.new(:attr_array, [params, nil, nil]) unless params.is_a?(Array) return if params.empty? - Filterly::Node.new(:attr_value, attr_array_params(params)) + Filterly::Node.new(:attr_array, attr_array_params(params)) end # @api private diff --git a/lib/filterly/pipeline/param_builder.rb b/lib/filterly/pipeline/param_builder.rb new file mode 100644 index 0000000..9d943c6 --- /dev/null +++ b/lib/filterly/pipeline/param_builder.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Filterly + module Pipeline + class ParamBuilder + attr_reader :filters, :from, :order, :params + + def initialize(filters:, from:, order:, params:) + @filters = filters + @from = from + @order = order + @params = params + end + + def limit + params[:limit] + end + + def offset + params[:offset] + end + + def search_query + params[:search_query] + end + + def self.new + super( + filters: Filterly::Tree.initialize_with_filters, + from: Set.new, + order: Set.new, + params: { limit: 10, offset: 0, search_query: nil } + ) + end + + def append(deps) + deps.each do |k, v| + case k + when :filters + @filters.prepend_ast(Filterly::Parser.new(v).to_ast, :and) + when :from + @from << v + when :order + @order << v + when :params + @params = @params.merge(v) + end + end + end + + alias << append + end + end +end diff --git a/lib/filterly/tree.rb b/lib/filterly/tree.rb index ce260b5..b95112e 100644 --- a/lib/filterly/tree.rb +++ b/lib/filterly/tree.rb @@ -4,11 +4,10 @@ module Filterly class Tree - attr_reader :root_node, :tree_traverser + attr_reader :root_node def initialize(root_node) @root_node = root_node - @tree_traverser = TreeTraverser.new(root_node) end def self.new(root_node) @@ -18,12 +17,23 @@ def self.new(root_node) super(root_node) end + def self.initialize_with_filters + new( + Filterly::NodeBuilder.build_custom_node( + type: :root, + args: [:filters, nil, nil] + ) + ) + end + def extend_ast(node_attr_name, new_node, stmt_type) - @root_node = tree_traverser.extend_ast(node_attr_name, new_node, stmt_type) + @root_node = TreeTraverser + .new(@root_node) + .extend_ast(node_attr_name, new_node, stmt_type) end def prepend_ast(new_node, stmt_type) - @root_node = tree_traverser.prepend_ast(new_node, stmt_type) + @root_node = TreeTraverser.new(@root_node).prepend_ast(new_node, stmt_type) end def to_ast @@ -73,13 +83,13 @@ def recreate_node(node_attr_name, new_node, stmt_type) [ ast_node.value, self - .class - .new(ast_node.left) - .extend_ast(node_attr_name, new_node, stmt_type), + .class + .new(ast_node.left) + .extend_ast(node_attr_name, new_node, stmt_type), self - .class - .new(ast_node.right) - .extend_ast(node_attr_name, new_node, stmt_type) + .class + .new(ast_node.right) + .extend_ast(node_attr_name, new_node, stmt_type) ] ) end diff --git a/spec/examples/interpreter_spec.rb b/spec/examples/interpreter_spec.rb index d491acf..49dfe63 100644 --- a/spec/examples/interpreter_spec.rb +++ b/spec/examples/interpreter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' require 'filterly/node' -require File.join Examples.root, 'interpreter' +require 'examples/interpreter' RSpec.describe Interpreter do subject do diff --git a/spec/examples/to_sql/expression_factory_spec.rb b/spec/examples/to_sql/expression_factory_spec.rb index be1fa80..15718a9 100644 --- a/spec/examples/to_sql/expression_factory_spec.rb +++ b/spec/examples/to_sql/expression_factory_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' require 'filterly/node' -require File.join Examples.root, 'interpreter/to_sql/expression_factory' +require 'examples/interpreter/to_sql/expression_factory' RSpec.describe Interpreter::ToSql::ExpressionFactory do subject do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 8cc261e..96eddbf 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift('.') + require "bundler/setup" require "filterly" @@ -12,9 +16,3 @@ c.syntax = :expect end end - -class Examples - def self.root - File.join (File.dirname __dir__), 'examples' - end -end diff --git a/spec/unit/node_builder_spec.rb b/spec/unit/node_builder_spec.rb index a0f97ac..683bf91 100644 --- a/spec/unit/node_builder_spec.rb +++ b/spec/unit/node_builder_spec.rb @@ -102,21 +102,22 @@ [:attr_name, ['course_id', [], []]], [ :attr_array, [ - nil, + 12, [ - :attr_value, [ - 12, - [:attr_value, [1, [], []]], - [ - :attr_value, [ - 3, - [:attr_value, [4, [], []]], - [] - ] - ] + :attr_array, [ + 1, + [], + [] ] ], - [] + [ + :attr_array, + [ + 3, + [:attr_array, [4, [], []]], + [] + ] + ] ] ] ] diff --git a/spec/unit/parser_spec.rb b/spec/unit/parser_spec.rb index cb9e56b..6707ed8 100644 --- a/spec/unit/parser_spec.rb +++ b/spec/unit/parser_spec.rb @@ -37,16 +37,16 @@ [ :attr_array, [ - nil, + 12, [ - :attr_value, + :attr_array, [ - 12, - [:attr_value, [34, [], []]], - [:attr_value, [54, [], []]] + 34, + [], + [] ] ], - [] + [:attr_array, [54, [], []]] ] ] ] diff --git a/spec/unit/pipeline/param_builder_spec.rb b/spec/unit/pipeline/param_builder_spec.rb new file mode 100644 index 0000000..4df34b9 --- /dev/null +++ b/spec/unit/pipeline/param_builder_spec.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'filterly/pipeline/param_builder' + +RSpec.describe Filterly::Pipeline::ParamBuilder do + subject do + described_class.new + end + + let(:new_params) do + { + filters: { + category_ids: [12, 34, 54], + course_semester_id: 123 + }, + from: 'courses', + order: { 'courses.id': 'asc' }, + params: { + limit: 10, + offset: 0, + search_query: 'adam', + not_supported: 'omitted' + }, + ingored_one: 'none' + } + end + + describe '#initialize' do + it 'creates root node for fitlers ast tree' do + expect(subject.filters.to_ast.to_a).to match_array([:root, [:filters, [], []]]) + end + end + + describe '#append' do + it 'appends filters, from, order and params while ignoring other keys' do + subject << new_params + subject << { filters: { course_timetable: '2017-03-03' } } + subject << { order: { 'another_col': 'desc' } } + subject << { params: { limit: 3 } } + + expect(subject.from).to match_array(['courses']) + expect(subject.limit).to eql(3) + expect(subject.offset).to eql(0) + expect(subject.search_query).to eql('adam') + + expect(subject.order).to match_array( + [ + { 'courses.id': 'asc' }, + { 'another_col': 'desc' } + ] + ) + + expect(subject.filters.root_node.to_a).to match_array( + [ + :root, + [ + :filters, + [ + :statement, + [ + :and, + [ + :expression, + [ + :op_equal, + [:attr_name, [:course_timetable, [], []]], + [:attr_value, ['2017-03-03', [], []]] + ] + ], + [ + :statement, + [ + :and, + [ + :statement, + [ + :and, + [ + :expression, + [ + :op_in, + [:attr_name, [:category_ids, [], []]], + [ + :attr_array, + [ + 12, + [:attr_array, [34, [], []]], + [:attr_array, [54, [], []]] + ] + ] + ] + ], + [ + :expression, + [ + :op_equal, + [:attr_name, [:course_semester_id, [], []]], + [:attr_value, [123, [], []]] + ] + ] + ] + ], + [] + ] + ] + ] + ], + [] + ] + ] + ) + end + end +end diff --git a/spec/unit/tree_spec.rb b/spec/unit/tree_spec.rb index fa8ea01..7614f66 100644 --- a/spec/unit/tree_spec.rb +++ b/spec/unit/tree_spec.rb @@ -83,6 +83,21 @@ end end + describe '#initialize_with_filters' do + it 'initializes Tree with filters root' do + expect(described_class.initialize_with_filters.root_node.to_a).to match_array( + [ + :root, + [ + :filters, + [], + [] + ] + ] + ) + end + end + describe '#extend_ast' do it 'adds a new node to specific node and returns root' do subject.extend_ast(node_attr_name, new_node, :or) From 683f2fe726851035760f7a1b2d53244a873b3c33 Mon Sep 17 00:00:00 2001 From: Tomek Date: Thu, 31 Aug 2017 11:28:59 +0200 Subject: [PATCH 2/2] Add pipeline example --- examples/interpreter/collector.rb | 18 ++++ .../interpreter/to_sql/expression_factory.rb | 2 +- examples/sql_builder.rb | 51 +++++++++++ examples/sql_struct_parts.rb | 50 +++++++++++ examples/user_pipeline.rb | 71 +++++++++++++++ examples/user_query_pipeline.rb | 30 +++++++ spec/examples/interpreter_spec.rb | 2 +- spec/examples/sql_builder_spec.rb | 87 +++++++++++++++++++ spec/examples/sql_struct_parts_spec.rb | 66 ++++++++++++++ .../to_sql/expression_factory_spec.rb | 2 +- 10 files changed, 376 insertions(+), 3 deletions(-) create mode 100644 examples/interpreter/collector.rb create mode 100644 examples/sql_builder.rb create mode 100644 examples/sql_struct_parts.rb create mode 100644 examples/user_pipeline.rb create mode 100644 examples/user_query_pipeline.rb create mode 100644 spec/examples/sql_builder_spec.rb create mode 100644 spec/examples/sql_struct_parts_spec.rb diff --git a/examples/interpreter/collector.rb b/examples/interpreter/collector.rb new file mode 100644 index 0000000..973dd41 --- /dev/null +++ b/examples/interpreter/collector.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require File.expand_path('../../interpreter', __FILE__) + +class Interpreter + class Collector + def call(param_builder) + { + where: [Interpreter.new(ast: param_builder.filters.to_ast).to_sql], + order: param_builder.order.first.to_a.join(' '), + from: param_builder.from.first, + limit: param_builder.limit, + offset: param_builder.offset, + search_query: param_builder.search_query + } + end + end +end diff --git a/examples/interpreter/to_sql/expression_factory.rb b/examples/interpreter/to_sql/expression_factory.rb index 4052e00..289adc6 100644 --- a/examples/interpreter/to_sql/expression_factory.rb +++ b/examples/interpreter/to_sql/expression_factory.rb @@ -32,7 +32,7 @@ def visit_course_category_ids(value, _left, right) <<~SQL EXISTS( SELECT TRUE FROM category_courses - WHERE category_courses.category_id #{determine_values(value, right)}) + WHERE category_courses.category_id #{determine_values(value, right)} AND courses.id = category_courses.course_id ) SQL diff --git a/examples/sql_builder.rb b/examples/sql_builder.rb new file mode 100644 index 0000000..5bd6a44 --- /dev/null +++ b/examples/sql_builder.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +class SqlBuilder + attr_reader :sql_struct_parts, :model_name + + def initialize(sql_struct_parts:, model_name:) + @sql_struct_parts = sql_struct_parts + @model_name = model_name + end + + def preloads + ssp.preloads + end + + def sql_query + <<~SQL + SELECT #{select} FROM #{from} #{ssp.joins.join(' ')} + WHERE #{ssp.where.join(' AND ')} + ORDER BY #{ssp.order} + LIMIT #{ssp.limit} + OFFSET #{ssp.offset} + SQL + end + + def count_query + <<~SQL + SELECT #{count_select} FROM #{from} #{ssp.joins.join(' ')} + WHERE #{ssp.where.join(' AND ')} + SQL + end + + def count_select + "COUNT(#{select})" + end + + def select + return '*' if ssp.select.empty? + ssp.select + end + + def from + return model_name if ssp.from.empty? + ssp.from + end + + def to_s + to_sql.to_s + end + + alias ssp sql_struct_parts +end diff --git a/examples/sql_struct_parts.rb b/examples/sql_struct_parts.rb new file mode 100644 index 0000000..26ec03f --- /dev/null +++ b/examples/sql_struct_parts.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +class SqlStructParts + PARTS = %w[select from joins where search_query order limit offset preload].freeze + + attr_reader :deps + + def initialize(**opts) + @deps = opts[:deps] || {} + end + + def with(deps) + self.class.new(deps: initialize_dependencies(deps)) + end + + PARTS.each do |sql_part| + define_method sql_part do + @deps[sql_part.to_sym] || [] + end + end + + # @api private + def initialize_dependencies(deps) + params = {} + params = initialize_arrays(params, deps) + params = initialize_constants(params, deps) + params + end + + # @api private + def initialize_arrays(params, deps) + params[:select] = select.to_a | deps[:select].to_a + params[:from] = deps[:from].to_s + params[:joins] = joins.to_a | deps[:joins].to_a + params[:where] = where | deps[:where].to_a + params + end + + # @api private + def initialize_constants(params, deps) + params[:search_query] = deps[:search_query].to_s + params[:order] = deps[:order].to_s + params[:limit] = deps[:limit].to_i + params[:offset] = deps[:offset].to_i + params[:preload] = preload | deps[:preload].to_a + params + end + + alias << with +end diff --git a/examples/user_pipeline.rb b/examples/user_pipeline.rb new file mode 100644 index 0000000..608f464 --- /dev/null +++ b/examples/user_pipeline.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'filterly' +require_relative 'interpreter/collector' +require_relative 'user_query_pipeline' + +class UserPipeline + attr_reader :params, :param_builder + + def initialize(params:, param_builder: ::Filterly::Pipeline::ParamBuilder.new) + @params = params + @param_builder = param_builder + end + + def call + build_params + + count, query = UserQueryPipeline.new(params: interpret_to_sql).call + puts '---------- SQL COUNT query -----------' + puts + puts count + puts + puts + puts '---------- SQL main query ------------' + puts query + puts + end + + # @api private + def build_params + param_builder << filters + param_builder << some_other_filters + end + + # @api private + def interpret_to_sql + Interpreter::Collector.new.call(param_builder) + end + + # @api private + def filters + params.to_h + end + + def some_other_filters + { + filters: { + course_ids: [12, 34, 54] + }, + select: ['courses.new_column as new_one'], + params: { search_query: 'adam' } + } + end +end + +UserPipeline.new( + params: { + filters: { + category_ids: [12, 34, 54], + course_annual_id: 23, + course_semester_id: 123 + }, + from: 'courses', + order: { 'courses.id': 'asc' }, + params: { + limit: 10, + offset: 0, + not_supported: 'omitted' + } + } +).call diff --git a/examples/user_query_pipeline.rb b/examples/user_query_pipeline.rb new file mode 100644 index 0000000..8bf7e39 --- /dev/null +++ b/examples/user_query_pipeline.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative 'sql_struct_parts' +require_relative 'sql_builder' + +class UserQueryPipeline + attr_reader :sql_struct_parts + + def initialize(sql_struct_parts:) + @sql_struct_parts = sql_struct_parts + end + + def self.new(params:) + super(sql_struct_parts: SqlStructParts.new(deps: params)) + end + + def call + sql_struct_parts << something_special + + sql_builder = SqlBuilder.new(sql_struct_parts: sql_struct_parts, model_name: 'users') + + [sql_builder.count_query, sql_builder.sql_query] + end + + def something_special + { + where: ["user.ethnicity IN('celts', 'slavic')"] + } + end +end diff --git a/spec/examples/interpreter_spec.rb b/spec/examples/interpreter_spec.rb index 49dfe63..9784105 100644 --- a/spec/examples/interpreter_spec.rb +++ b/spec/examples/interpreter_spec.rb @@ -128,7 +128,7 @@ (course_id='23' OR (course_id='7' OR course_id='56')) AND annual='2017-2018' AND EXISTS( SELECT TRUE FROM category_courses - WHERE category_courses.category_id IN('67','32','34')) + WHERE category_courses.category_id IN('67','32','34') AND courses.id = category_courses.course_id ) SQL diff --git a/spec/examples/sql_builder_spec.rb b/spec/examples/sql_builder_spec.rb new file mode 100644 index 0000000..b434dee --- /dev/null +++ b/spec/examples/sql_builder_spec.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require 'examples/sql_builder' +require 'examples/sql_struct_parts' + +RSpec.describe SqlBuilder do + subject do + described_class.new(sql_struct_parts: sql_struct, model_name: 'users') + end + + let(:sql_struct) do + SqlStructParts.new( + deps: { + select: 'users.*', + from: '(SELECT users WHERE id IN(1,2,3)) as users', + where: ['(users.name = "Pszemek" OR users.age > 24)', 'users.status = "adult"'], + joins: ['LEFT JOIN courses c ON(c.id=users.course_id)'], + limit: 10, + order: 'users.id ASC', + offset: 5, + preload: %i[courses addresses] + } + ) + end + + let(:sql_struct_no_from_or_select) do + SqlStructParts.new( + deps: { + select: [], + from: nil, + where: ['users.name = "Pszemek" AND users.age > 24'], + joins: ['LEFT JOIN courses c ON(c.id=users.course_id)'], + limit: 10, + order: 'users.id ASC', + offset: 5, + preload: %i[courses addresses] + } + ) + end + + describe '#sql_query' do + it 'returns sql query' do + expect(subject.sql_query.tr("\n", ' ')).to eql( + <<~SQL.tr("\n", ' ') + SELECT users.* FROM (SELECT users WHERE id IN(1,2,3)) as users + LEFT JOIN courses c ON(c.id=users.course_id) + WHERE (users.name = "Pszemek" OR users.age > 24) AND users.status = "adult" + ORDER BY users.id ASC + LIMIT 10 + OFFSET 5 + SQL + ) + end + + it 'uses model_name when from is upsent' do + result = described_class.new( + sql_struct_parts: sql_struct_no_from_or_select, + model_name: 'users' + ).sql_query + + expect(result.tr("\n", ' ')).to eql( + <<~SQL.tr("\n", ' ') + SELECT * FROM users + LEFT JOIN courses c ON(c.id=users.course_id) + WHERE users.name = "Pszemek" AND users.age > 24 + ORDER BY users.id ASC + LIMIT 10 + OFFSET 5 + SQL + ) + end + end + + describe '#count_query' do + it 'returns sql count query' do + expect(subject.count_query.tr("\n", ' ')).to eql( + <<~SQL.tr("\n", ' ') + SELECT COUNT(users.*) FROM (SELECT users WHERE id IN(1,2,3)) as users + LEFT JOIN courses c ON(c.id=users.course_id) + WHERE (users.name = "Pszemek" OR users.age > 24) AND users.status = "adult" + SQL + ) + end + end +end diff --git a/spec/examples/sql_struct_parts_spec.rb b/spec/examples/sql_struct_parts_spec.rb new file mode 100644 index 0000000..f0c99ad --- /dev/null +++ b/spec/examples/sql_struct_parts_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' +require './examples/sql_struct_parts' + +RSpec.describe SqlStructParts do + subject do + described_class.new(deps: deps) + end + + let(:deps) do + { + select: ['users.*'], + where: ['user.id IN(1,2,3)'], + limit: 20, + joins: ['LEFT JOIN courses c ON(c.id=b.lk)'] + } + end + + describe '#new' do + it 'creates deps with default values' do + expect(subject.deps).to eql( + select: ['users.*'], + limit: 20, + where: ['user.id IN(1,2,3)'], + joins: ['LEFT JOIN courses c ON(c.id=b.lk)'] + ) + end + end + + describe '#with' do + it 'merges new params with the old ones' do + result = subject + .with( + limit: 10, + offset: 0, + joins: ['INNER JOIN addresses a ON(a.id=b.address_id)'] + ) + .with(from: 'users', select: ['id, surname'], where: ['user.age > 45']) + + expect(result.deps).to eql( + from: 'users', + joins: [ + 'LEFT JOIN courses c ON(c.id=b.lk)', + 'INNER JOIN addresses a ON(a.id=b.address_id)' + ], + limit: 0, + offset: 0, + order: '', + preload: [], + search_query: '', + select: ['users.*', 'id, surname'], + where: ['user.id IN(1,2,3)', 'user.age > 45'] + ) + end + end + + describe '#initialize_dependencies' do + it 'allows to call attributes by methods' do + result = subject.with(limit: 11) + + expect(result.limit).to eql(11) + expect(result.where).to eql(['user.id IN(1,2,3)']) + end + end +end diff --git a/spec/examples/to_sql/expression_factory_spec.rb b/spec/examples/to_sql/expression_factory_spec.rb index 15718a9..b993246 100644 --- a/spec/examples/to_sql/expression_factory_spec.rb +++ b/spec/examples/to_sql/expression_factory_spec.rb @@ -44,7 +44,7 @@ <<~SQL.split.join(' ') EXISTS( SELECT TRUE FROM category_courses - WHERE category_courses.category_id IN('67','32','34')) + WHERE category_courses.category_id IN('67','32','34') AND courses.id = category_courses.course_id ) SQL