Skip to content
Snippets Groups Projects
Commit 63f0909f authored by Nia Kathoni's avatar Nia Kathoni Committed by Daniel Cothran
Browse files

Issue #3501561 by nikathone: Select filter broken when header offset is set

parent 9aa1f5b9
No related branches found
No related tags found
1 merge request!24Select filter broken when header offset is set
Pipeline #404026 passed with warnings
......@@ -7,7 +7,6 @@ use Drupal\Core\DependencyInjection\DependencySerializationTrait;
use League\Csv\Reader;
use League\Csv\ResultSet;
use League\Csv\Statement;
use League\Csv\TabularDataReader;
/**
* Defines a class for building and executing a select query.
......@@ -51,11 +50,6 @@ class Select {
*/
protected array $groupedAndAggregatedRecords = [];
/**
* The selected column headers.
*/
protected array $selectedColumnHeaders = [];
/**
* Flag for whether execute() was already called for this query.
*
......@@ -438,14 +432,18 @@ class Select {
}
$uri = $this->connection->parseCsvUri();
$default_csv_options = $this->getDefaultCsvOptions();
$csv_content = $this->connection->fetchContent($uri, $default_csv_options);
$options = $this->queryOptions += [
'delimiter' => ',',
'header_index' => 0,
];
$csv_content = $this->connection->fetchContent($uri, $options);
if (!$csv_content) {
return [];
}
$csv = Reader::createFromString($csv_content);
$csv->setHeaderOffset($default_csv_options['header_index']);
$csv->setHeaderOffset($options['header_index']);
$csv->setDelimiter($options['delimiter']);
// Process CSV Headers.
// @todo maybe find a way to remove duplicates? Right now only removing
......@@ -453,67 +451,44 @@ class Select {
if (!($all_headers = $csv->getHeader())) {
return [];
}
$this->selectedColumnHeaders = array_intersect(array_filter($all_headers), $field_keys);
$csv->mapHeader($this->selectedColumnHeaders);
// Initialize other reader options.
$csv = $this->initializeCsvOptions($csv);
$stmt = Statement::create();
// 1. Map headers to ensure that we only keep the column we need.
$selected_column_headers = array_intersect(array_filter($all_headers), $field_keys);
$result_set = $this->mapHeaders($csv, $selected_column_headers, $options['header_index']);
// 1. Adding filter clause(s) to the statement.
$stmt = $this->applyFilters($stmt);
// 2. Filter the result set.
$result_set = $this->applyFilters($result_set);
if ($result_set->count() === 0) {
return [];
}
// 2. Apply Sort.
// 3. Sort the result set if this is not a count query.
if (!$this->isCountQuery()) {
// Only applying sorting for a non-counting query.
$stmt = $this->applyOrderBy($stmt);
$result_set = $this->applyOrderBy($result_set);
}
// 3. Apply grouping here.
// 4. Group the result set and create a new one based on the grouping.
if ($group_by = $this->getObjectItem('group_by')) {
// Processing filtered csv to reduce the number of records to be grouped
// if necessary.
$result_set = $stmt->process($csv, $this->selectedColumnHeaders);
if ($result_set->count() === 0) {
return [];
}
// Reset the statement to remove the filter clauses added in #1.
$stmt = Statement::create();
$result_set = $this->buildFilteredResultSet($result_set->getHeader(), $result_set->getRecords());
// Applying a group by and aggregation callback.
$result_set = $this->applyGroupBy($result_set, $group_by, $this->getSelectedAggregatedFields());
// Initialize a new statement that doesn't have any of the filtering
// closures.
$group_by_stmt = Statement::create();
// Process the grouping.
$group_by_stmt->process($result_set, $this->selectedColumnHeaders);
}
// 4. Flatten the group by and aggregation array to build a new csv reader.
if ($this->groupedAndAggregatedRecords) {
$csv = $this->buildCsvReader();
// 5. No need for offset and limit on a count query.
if ($this->isCountQuery()) {
$this->records = $result_set->getRecords();
return $this->records;
}
// 5. Apply offset and limit.
// 6. Apply offset and limit.
$stmt = Statement::create();
$offset = $this->getOffset();
if ($this->getDefaultCsvOptions()['header_index']) {
$offset += $this->getDefaultCsvOptions()['header_index'];
}
if ($offset > 0) {
$stmt = $stmt->offset($offset);
}
$limit = $this->getLimit();
if ($limit > 0) {
$stmt = $stmt->limit($limit);
}
$result_set = $stmt->process($csv, $this->selectedColumnHeaders);
$result_set = $stmt->process($result_set);
$this->records = $result_set->getRecords();
return $this->records;
}
......@@ -529,33 +504,70 @@ class Select {
}
/**
* Apply the filters to the statement.
* Maps the headers to a CSV reader.
*
* @param \League\Csv\Statement $stmt
* The statement to apply the filters to.
* @param \League\Csv\Reader $reader
* The CSV reader to map the headers to.
* @param array $headers
* The headers to map to the CSV reader.
* @param int $header_index
* The header index to user for offset.
*
* @return \League\Csv\ResultSet
* A new result set with the headers mapped to it.
*
* @return \League\Csv\Statement
* The statement with the filters applied.
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
* @throws \League\Csv\SyntaxError
* @throws \ReflectionException
*/
protected function applyFilters(Statement $stmt): Statement {
private function mapHeaders(Reader $reader, array $headers, int $header_index): ResultSet {
// Using the header index to offset some of the rows that might not be
// needed.
return Statement::create()->offset($header_index)->process($reader, $headers);
}
/**
* Filters the result set.
*
* @param \League\Csv\ResultSet $result_set
* The result set to filter.
*
* @return \League\Csv\ResultSet
* A filtered result set if where condition were added to the select query.
*
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
* @throws \League\Csv\SyntaxError
* @throws \ReflectionException
*/
protected function applyFilters(ResultSet $result_set): ResultSet {
$stmt = Statement::create();
if ($where = $this->getObjectItem('where')) {
return $stmt->where(function (array $record) use ($where): bool {
$stmt = $stmt->where(function (array $record) use ($where): bool {
return $record && static::conditionPassed($record, (array) $where['conditions'], $where['conjunction']);
});
}
return $stmt;
return $stmt->process($result_set);
}
/**
* Apply the order by to the statement.
* Orders the result set.
*
* @param \League\Csv\ResultSet $result_set
* The result set to order.
*
* @param \League\Csv\Statement $stmt
* The statement to apply the order by to.
* @return \League\Csv\ResultSet
* An ordered result set if "order by" was added to the query.
*
* @return \League\Csv\Statement
* The statement with the order by applied.
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
* @throws \League\Csv\SyntaxError
* @throws \ReflectionException
*/
protected function applyOrderBy(Statement $stmt): Statement {
protected function applyOrderBy(ResultSet $result_set): ResultSet {
$stmt = Statement::create();
if ($order_by = $this->getObjectItem('order_by')) {
foreach ($order_by as $sort) {
$stmt = $stmt->orderBy(function (array $a, array $b) use ($sort): int {
......@@ -567,26 +579,30 @@ class Select {
};
});
}
return $stmt;
}
return $stmt;
return $stmt->process($result_set);
}
/**
* Apply the group by and aggregation.
* Aggregates and groups the result set.
*
* @param \League\Csv\TabularDataReader $result_set
* The result set to apply the group by and aggregation to.
* @param \League\Csv\ResultSet $result_set
* The set of results to group and aggregate.
* @param array $group_by_columns
* The names of the columns to group by.
* @param array $aggregated_columns
* The names of the columns to aggregate.
*
* @return \League\Csv\TabularDataReader
* The result set with the group by and aggregation applied.
* @return \League\Csv\ResultSet
* A new result set that has been grouped and aggregated.
*
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
* @throws \League\Csv\SyntaxError
* @throws \ReflectionException
*/
protected function applyGroupBy(TabularDataReader $result_set, array $group_by_columns, array $aggregated_columns): TabularDataReader {
// Ensure that all the columns that the grouping is going to apply to
protected function applyGroupBy(ResultSet $result_set, array $group_by_columns, array $aggregated_columns): ResultSet {
// Ensure that all the columns that the grouping is going to apply to are
// existing columns.
if ($group_by_columns !== array_intersect($group_by_columns, $result_set->getHeader())) {
return $result_set;
......@@ -595,13 +611,39 @@ class Select {
$result_set->each(function ($row) use ($group_by_columns, $aggregated_columns) {
$grouped_by_values = static::extractGroupedColumnsValuesFromRecord($row, $group_by_columns);
static::processGroupByAndAggregation($grouped_by_values, $group_by_columns, $aggregated_columns, $row);
return TRUE;
});
return $result_set;
// Trigger the "each" method predicate so that the
// "groupedAndAggregatedRecords" property can be populated.
Statement::create()->process($result_set);
if (!$this->groupedAndAggregatedRecords) {
return $result_set;
}
$records = [];
$headers = [];
foreach ($this->groupedAndAggregatedRecords as $record) {
if (!$headers) {
$headers = array_keys($record);
}
$records[] = array_values($record);
}
$tmp = new \SplTempFileObject();
$tmp->fputcsv($headers);
foreach ($records as $record) {
$tmp->fputcsv($record);
}
$csv = Reader::createFromFileObject($tmp);
$delimiter = $this->queryOptions['delimiter'] ?? ',';
$csv->setDelimiter($delimiter);
$csv->setHeaderOffset(0);
return Statement::create()->process($csv, $headers);
}
/**
* Add a column to the fields array.
* Add a column to the "fields" property array.
*
* @param \stdClass $column
* The column to add.
......@@ -858,30 +900,6 @@ class Select {
return $fields;
}
/**
* Build a CSV reader from the grouped and aggregated records.
*
* @return \League\Csv\Reader
* The CSV reader.
*
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
*/
protected function buildCsvReader(): Reader {
$records = [];
$headers = [];
foreach ($this->groupedAndAggregatedRecords as $record) {
if (!$headers) {
$headers = array_keys($record);
}
$records[] = array_values($record);
}
$this->selectedColumnHeaders = $headers;
return $this->buildNewCsvFromRecords($headers, $records);
}
/**
* Process the group by and aggregation.
*
......@@ -936,89 +954,6 @@ class Select {
}
}
/**
* Build a new CSV reader from records.
*
* @param array $header
* The header to assign to the CSV.
* @param array|\Iterator $records
* The records to add to the CSV.
*
* @return \League\Csv\Reader
* The new CSV reader.
*
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
*/
protected function buildNewCsvFromRecords(array $header, array|\Iterator $records): Reader {
$tmp = new \SplTempFileObject();
$tmp->fputcsv($header);
foreach ($records as $record) {
$tmp->fputcsv($record);
}
$csv = Reader::createFromFileObject($tmp);
return $this->initializeCsvOptions($csv, ['header_index' => 0]);
}
/**
* Build the result set from the filtered result set records.
*
* @param array $header
* The header of the new CSV.
* @param \Iterator $records
* The filtered records.
*
* @return \League\Csv\ResultSet
* The result set.
*
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
* @throws \League\Csv\SyntaxError
* @throws \ReflectionException
*/
protected function buildFilteredResultSet(array $header, \Iterator $records): ResultSet {
$csv = $this->buildNewCsvFromRecords($header, $records);
$stmt = Statement::create();
return $stmt->process($csv);
}
/**
* Initialize the CSV reader with the default options.
*
* @param \League\Csv\Reader $csv
* The CSV reader to initialize.
* @param array $options
* Options to overwrite default options during initialization.
*
* @return \League\Csv\Reader
* The initialized CSV reader.
*
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
*/
protected function initializeCsvOptions(Reader $csv, array $options = []): Reader {
$options += $this->getDefaultCsvOptions();
$csv->setDelimiter($options['delimiter']);
if (isset($options['header_index'])) {
$csv->setHeaderOffset($options['header_index']);
$this->queryOptions['header_index'] = $options['header_index'];
}
return $csv;
}
/**
* Default CSV and query options.
*
* @return array
* The options.
*/
protected function getDefaultCsvOptions(): array {
return $this->queryOptions += [
'delimiter' => ',',
'header_index' => 0,
];
}
/**
* Combines "in" and "contains" operator to check both on an array condition.
*
......
......@@ -246,8 +246,6 @@ class SelectTest extends UnitTestCase {
* @covers ::getRecords
* @covers ::extractGroupedColumnsValuesFromRecord
* @covers ::processGroupByAndAggregation
* @covers ::buildCsvReader
* @covers ::buildNewCsvFromRecords
* @covers ::applyOrderBy
*/
public function testGroupByWithOrderBy() {
......@@ -376,6 +374,7 @@ class SelectTest extends UnitTestCase {
* @covers ::groupBy
* @covers ::getRecords
* @covers ::applyOrderBy
*
* @throws \League\Csv\Exception
*/
public function testGroupByAndAggregationOnSameColumnWithAlias() {
......@@ -398,6 +397,7 @@ class SelectTest extends UnitTestCase {
/**
* @covers ::getRecords
* @covers ::getSelectedAggregatedFields
*
* @throws \League\Csv\Exception
*/
public function testCsvWithEmptyColumns() {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment