Skip to content
Snippets Groups Projects
Commit 56d64e28 authored by Daniel Cothran's avatar Daniel Cothran
Browse files

Issue #3498258 by nikathone, andileco: Add interface for ignoring header rows

parent 8c1d02ba
No related branches found
No related tags found
1 merge request!22first attempt - select filter still not working
Pipeline #397065 passed with warnings
......@@ -65,6 +65,9 @@ views.query.views_csv_source_query:
headers:
type: string
label: 'Headers'
header_index:
type: integer
label: 'Header offset'
request_method:
type: string
label: 'Request method'
......
......@@ -327,6 +327,7 @@ class ViewsCsvQuery extends QueryPluginBase {
public function defineOptions(): array {
$options = parent::defineOptions();
$options['csv_file'] = ['default' => ''];
$options['header_index'] = ['default' => 0];
$options['headers'] = ['default' => ''];
$options['request_method'] = ['default' => 'get'];
$options['request_body'] = ['default' => ''];
......@@ -366,6 +367,15 @@ class ViewsCsvQuery extends QueryPluginBase {
'#element_validate' => [[static::class, 'validateCsvFileUriElement']],
'#required' => TRUE,
];
$form['header_index'] = [
'#title' => $this->t('Header offset'),
'#type' => 'number',
'#min' => 0,
'#step' => 1,
'#required' => FALSE,
'#default_value' => $this->options['header_index'],
'#description' => $this->t('The number of rows to skip before the header row.'),
];
$form['headers'] = [
'#type' => 'textfield',
'#title' => $this->t('Headers'),
......@@ -806,6 +816,10 @@ class ViewsCsvQuery extends QueryPluginBase {
else {
$query_options['headers'] = Json::decode($query_options['headers']);
}
if (!empty($this->options['header_index'])) {
$query_options['header_index'] = (int) $this->options['header_index'];
}
return $query_options;
}
......
......@@ -227,7 +227,10 @@ class Connection {
try {
$csv = Reader::createFromString($this->fetchContent($uri, $options));
$headers = $csv->nth(0) ?? [];
// Avoid warnings in PHP 8.4.
$csv->setEscape('');
$csv->setHeaderOffset($options['header_index']);
$headers = $csv->getHeader() ?? [];
$this->csvHeader[$uri] = $headers ? array_filter($headers) : [];
return $this->csvHeader[$uri];
}
......@@ -251,7 +254,7 @@ class Connection {
*/
public function getCsvColumnData(string $uri = '', string $header = '', array $query_options = []): array {
$uri = empty($uri) ? $this->csvUri : $uri;
if (empty($uri)) {
if (empty($uri) || !$header) {
return [];
}
......@@ -262,8 +265,24 @@ class Connection {
try {
$csv = Reader::createFromString($this->fetchContent($uri, $options));
$csv->setHeaderOffset(0);
$records = Statement::create()->process($csv);
$csv->setHeaderOffset($options['header_index']);
$all_headers = $csv->getHeader();
// Ensuring that the selected header is part of CSV headers and removing
// empty column headers.
$selected_headers = array_intersect(array_filter($all_headers), [$header]);
if (!$selected_headers) {
return [];
}
$csv->mapHeader($selected_headers);
$stmt = Statement::create();
// Adding offset when header index is not on the first row.
if ($options['header_index'] > 0) {
$stmt = $stmt->offset($options['header_index']);
}
$records = $stmt->process($csv, $selected_headers);
$column_contents = [];
foreach ($records->fetchColumnByName($header) as $column_content) {
$column_contents[] = $column_content;
......@@ -271,6 +290,7 @@ class Connection {
return array_unique($column_contents);
}
catch (\Exception $e) {
$this->logger->error($e->getMessage());
return [];
}
}
......
......@@ -438,18 +438,19 @@ class Select {
}
$uri = $this->connection->parseCsvUri();
$csv_content = $this->connection->fetchContent($uri, $this->getDefaultCsvOptions());
$default_csv_options = $this->getDefaultCsvOptions();
$csv_content = $this->connection->fetchContent($uri, $default_csv_options);
if (!$csv_content) {
return [];
}
$csv = Reader::createFromString($csv_content);
$csv->setHeaderOffset($default_csv_options['header_index']);
// Process CSV Headers.
// @todo maybe find a way to remove duplicates? Right now only removing
// empty columns.
$all_headers = $csv->nth(0);
if (!$all_headers) {
if (!($all_headers = $csv->getHeader())) {
return [];
}
$this->selectedColumnHeaders = array_intersect(array_filter($all_headers), $field_keys);
......@@ -498,11 +499,18 @@ class Select {
$csv = $this->buildCsvReader();
}
// 5. Apply range.
// 5. Apply offset and limit.
$offset = $this->getOffset();
if ($this->getDefaultCsvOptions()['header_index']) {
$offset += $this->getDefaultCsvOptions()['header_index'];
}
if ($offset > 0) {
$stmt = $stmt->offset($offset);
}
$limit = $this->getLimit();
if ($limit > 0) {
$stmt = $stmt->limit($limit)->offset($offset);
$stmt = $stmt->limit($limit);
}
$result_set = $stmt->process($csv, $this->selectedColumnHeaders);
......@@ -949,7 +957,7 @@ class Select {
$tmp->fputcsv($record);
}
$csv = Reader::createFromFileObject($tmp);
return $this->initializeCsvOptions($csv);
return $this->initializeCsvOptions($csv, ['header_index' => 0]);
}
/**
......@@ -979,6 +987,8 @@ class Select {
*
* @param \League\Csv\Reader $csv
* The CSV reader to initialize.
* @param array $options
* Options to overwrite default options during initialization.
*
* @return \League\Csv\Reader
* The initialized CSV reader.
......@@ -986,11 +996,12 @@ class Select {
* @throws \League\Csv\Exception
* @throws \League\Csv\InvalidArgument
*/
protected function initializeCsvOptions(Reader $csv): Reader {
$options = $this->getDefaultCsvOptions();
protected function initializeCsvOptions(Reader $csv, array $options = []): Reader {
$options += $this->getDefaultCsvOptions();
$csv->setDelimiter($options['delimiter']);
if (isset($options['header_index']) && $options['header_index'] !== NULL) {
if (isset($options['header_index'])) {
$csv->setHeaderOffset($options['header_index']);
$this->queryOptions['header_index'] = $options['header_index'];
}
return $csv;
}
......
......@@ -69,6 +69,7 @@ display:
request_method: get
request_body: ''
show_errors: true
header_index: 0
page_1:
id: page_1
display_title: 'Page display'
......
These,Are,Not Real,Headers
Geography name,Occupation Name,Total count,Keywords
Alabama,Counselors,34,Counselors
Alabama,Massage Therapists,67,Therapists
Alaska,Dental Hygienists,8989,Hygienists
Alaska,Emergency Medical Technicians and Paramedics,56,"Technicians, Paramedics"
Hawaii,Health Practitioner Support Technologists and Technicians,543,"Technologists, Technicians"
Illinois,Dental Assistants,76,Assistants
Kansas,Emergency Medical Technicians and Paramedics,76,"Technicians, Paramedics"
......@@ -63,22 +63,39 @@ class ViewsCsvSourceViewTest extends ViewTestBase {
* Test views csv source.
*/
public function testViewsCsvSource() {
$this->drupalGet('test-views-csv-source');
// Check that the page can be loaded.
$this->assertSession()->statusCodeEquals(Response::HTTP_OK);
// Testing regular view and header index.
$file = File::load(1);
foreach (['regular', 'header_index'] as $test_type) {
if ($test_type === 'header_index') {
file_put_contents($file->getFileUri(), file_get_contents($this->retrieveResource('/views_csv_data_with_header_index_moved_test.csv')));
$view = Views::getView('test_views_csv_source');
$display = &$view->storage->getDisplay('default');
$display['display_options']['query']['options']['header_index'] = 1;
$view->save();
}
$this->drupalGet('test-views-csv-source');
// Check that the page can be loaded.
$this->assertSession()->statusCodeEquals(Response::HTTP_OK);
$this->assertSession()->elementsCount('xpath', '//thead/tr/th[text()="Geography name"]', 1);
$this->assertSession()->elementsCount('xpath', '//thead/tr/th[text()="Occupation name"]', 1);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Alabama")]', 2);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Alaska")]', 2);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Hawaii")]', 1);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Illinois")]', 1);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Kansas")]', 1);
}
$this->assertSession()->elementsCount('xpath', '//thead/tr/th[text()="Geography name"]', 1);
$this->assertSession()->elementsCount('xpath', '//thead/tr/th[text()="Occupation name"]', 1);
$view = Views::getView('test_views_csv_source');
$display = &$view->storage->getDisplay('default');
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Alabama")]', 2);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Alaska")]', 2);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Hawaii")]', 1);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Illinois")]', 1);
$this->assertSession()->elementsCount('xpath', '//tbody/tr/td[starts-with(text(), "Kansas")]', 1);
// Set header index back to 0.
$display['display_options']['query']['options']['header_index'] = 0;
file_put_contents($file->getFileUri(), file_get_contents($this->retrieveResource()));
// Testing full pager.
$view = Views::getView('test_views_csv_source');
$display = &$view->storage->getDisplay('default');
$display['display_options']['pager']['options']['items_per_page'] = 3;
$display['display_options']['pager']['type'] = 'full';
$display['display_options']['pager']['options']['tags']['next'] = 'Next >';
......
......@@ -74,16 +74,22 @@ class SelectTest extends UnitTestCase {
* @covers ::addColumn
* @covers ::execute
* @covers ::getRecords
*
* @dataProvider executeWithFieldsDataProvider
*/
public function testExecuteWithFields() {
$select = $this->getSelectQuery()
public function testExecuteWithFields(array $options_overrides, int $expected_count, $expected_result_index, array $range) {
$select = $this->getSelectQuery($options_overrides)
->addField('Geography name')
->addField('Occupation Name');
if (!empty($range['offset'])) {
$select->range($range['offset'], $range['limit']);
}
$records = $select->execute();
$records = array_values(iterator_to_array($records));
$this->assertCount(7, $records);
$this->assertCount($expected_count, $records);
$this->assertArrayHasKey('Geography name', $records[0]);
$this->assertArrayHasKey('Occupation Name', $records[0]);
......@@ -92,7 +98,10 @@ class SelectTest extends UnitTestCase {
$this->assertEquals([
'Geography name' => 'Hawaii',
'Occupation Name' => 'Health Practitioner Support Technologists and Technicians',
], $records[4]);
], $records[$expected_result_index]);
// Restore the proper csv uri just in case it's cached.
$this->csvUri = $this->retrieveResource();
}
/**
......@@ -510,14 +519,43 @@ class SelectTest extends UnitTestCase {
];
}
/**
* Data provider for ::executeWithFields().
*/
public function executeWithFieldsDataProvider(): array {
$count = 7;
$header_index_overrides = [
'header_index' => 1,
'csv_file' => $this->retrieveResource('/views_csv_data_with_header_index_moved_test.csv')
];
return [
'no_overrides' => [[], $count, 4, []],
'header_index_overrides' => [
$header_index_overrides,
$count,
4,
[],
],
'header_index_overrides_with_range' => [
$header_index_overrides,
4,
2,
['offset' => 2, 'limit' => 4],
],
];
}
/**
* Gets the select query object.
*
* @return \Drupal\views_csv_source\Query\Select
* The select query object.
*/
protected function getSelectQuery(): Select {
return $this->connection->select($this->csvUri, [
protected function getSelectQuery(array $options = []): Select {
if (!empty($options['csv_file'])) {
$this->csvUri = $options['csv_file'];
}
return $this->connection->select($this->csvUri, $options + [
'headers' => ['Content-Type' => 'application/csv'],
'cache_id' => 'views_csv_source_data_test',
'csv_file' => $this->csvUri,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment