Work on setting up locale-aware formatted number conversion for the Csv Reader

Unit tests for locale-aware boolean conversion for Csv Reader

CHANGELOG.md

@@ -9,7 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Added
 
-- Implementation of the ISREF() information function
+- Implementation of the ISREF() information function.
+- Allow Boolean Conversion in Csv Reader to be locale-aware when using the String Value Binder.
+
+  (i.e. `"Vrai"` wil be converted to a boolean `true` if the Locale is set to `fr`.)
 
 ### Changed
 
@@ -27,7 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Fixed
 
-- Fixed behaviour of XLSX font style vertical align settings
+- Fixed behaviour of XLSX font style vertical align settings.
 - Resolved formula translations to handle separators (row and column) for array functions as well as for function argument separators; and cleanly handle nesting levels.
 
   Note that this method is used when translating Excel functions between en and other locale languages, as well as when converting formulae between different spreadsheet formats (e.g. Ods to Excel).

src/PhpSpreadsheet/Reader/Csv.php

@@ -9,6 +9,7 @@ use PhpOffice\PhpSpreadsheet\Reader\Csv\Delimiter;
 use PhpOffice\PhpSpreadsheet\Reader\Exception as ReaderException;
 use PhpOffice\PhpSpreadsheet\Shared\StringHelper;
 use PhpOffice\PhpSpreadsheet\Spreadsheet;
+use PhpOffice\PhpSpreadsheet\Style\NumberFormat;
 
 class Csv extends BaseReader
 {
@@ -85,6 +86,16 @@ class Csv extends BaseReader
      */
     private static $constructorCallback;
 
+    /**
+     * @var bool
+     */
+    protected $castFormattedNumberToNumeric = false;
+
+    /**
+     * @var bool
+     */
+    protected $preserveNumericFormatting = false;
+
     /**
      * Create a new CSV Reader instance.
      */
@@ -283,6 +294,14 @@ class Csv extends BaseReader
         return $retVal;
     }
 
+    public function castFormattedNumberToNumeric(
+        bool $castFormattedNumberToNumeric,
+        bool $preserveNumericFormatting = false
+    ): void {
+        $this->castFormattedNumberToNumeric = $castFormattedNumberToNumeric;
+        $this->preserveNumericFormatting = $preserveNumericFormatting;
+    }
+
     /**
      * Loads PhpSpreadsheet from file into PhpSpreadsheet instance.
      */
@@ -319,6 +338,7 @@ class Csv extends BaseReader
             $columnLetter = 'A';
             foreach ($rowData as $rowDatum) {
                 $this->convertBoolean($rowDatum, $preserveBooleanString);
+                $numberFormatMask = $this->convertFormattedNumber($rowDatum);
                 if ($rowDatum !== '' && $this->readFilter->readCell($columnLetter, $currentRow)) {
                     if ($this->contiguous) {
                         if ($noOutputYet) {
@@ -328,6 +348,10 @@ class Csv extends BaseReader
                     } else {
                         $outRow = $currentRow;
                     }
+                    // Set basic styling for the value (Note that this could be overloaded by styling in a value binder)
+                    $sheet->getCell($columnLetter . $outRow)->getStyle()
+                        ->getNumberFormat()
+                        ->setFormatCode($numberFormatMask);
                     // Set cell value
                     $sheet->getCell($columnLetter . $outRow)->setValue($rowDatum);
                 }
@@ -364,6 +388,39 @@ class Csv extends BaseReader
         }
     }
 
+    /**
+     * Convert numeric strings to int or float values.
+     *
+     * @param mixed $rowDatum
+     */
+    private function convertFormattedNumber(&$rowDatum): string
+    {
+        $numberFormatMask = NumberFormat::FORMAT_GENERAL;
+        if ($this->castFormattedNumberToNumeric === true && is_string($rowDatum)) {
+            $numeric = str_replace(
+                [StringHelper::getThousandsSeparator(), StringHelper::getDecimalSeparator()],
+                ['', '.'],
+                $rowDatum
+            );
+
+            if (is_numeric($numeric)) {
+                $decimalPos = strpos($rowDatum, StringHelper::getDecimalSeparator());
+                if ($this->preserveNumericFormatting === true) {
+                    $numberFormatMask = (strpos($rowDatum, StringHelper::getThousandsSeparator()) !== false)
+                        ? '#,##0' : '0';
+                    if ($decimalPos !== false) {
+                        $decimals = strlen($rowDatum) - $decimalPos - 1;
+                        $numberFormatMask .= '.' . str_repeat('0', min($decimals, 6));
+                    }
+                }
+
+                $rowDatum = ($decimalPos !== false) ? (float) $numeric : (int) $numeric;
+            }
+        }
+
+        return $numberFormatMask;
+    }
+
     public function getDelimiter(): ?string
     {
         return $this->delimiter;

tests/PhpSpreadsheetTests/Reader/Csv/CsvIssue2232Test.php

@@ -2,6 +2,7 @@
 
 namespace PhpOffice\PhpSpreadsheetTests\Reader\Csv;
 
+use PhpOffice\PhpSpreadsheet\Calculation\Calculation;
 use PhpOffice\PhpSpreadsheet\Cell\Cell;
 use PhpOffice\PhpSpreadsheet\Cell\IValueBinder;
 use PhpOffice\PhpSpreadsheet\Cell\StringValueBinder;
@@ -31,7 +32,7 @@ class CsvIssue2232Test extends TestCase
      * @param mixed $b2Value
      * @param mixed $b3Value
      */
-    public function testEncodings(bool $useStringBinder, ?bool $preserveBoolString, $b2Value, $b3Value): void
+    public function testBooleanConversions(bool $useStringBinder, ?bool $preserveBoolString, $b2Value, $b3Value): void
     {
         if ($useStringBinder) {
             $binder = new StringValueBinder();
@@ -60,4 +61,41 @@ class CsvIssue2232Test extends TestCase
             [true, true, 'FaLSe', 'tRUE'],
         ];
     }
+
+    /**
+     * @dataProvider providerIssue2232locale
+     *
+     * @param mixed $b4Value
+     * @param mixed $b5Value
+     */
+    public function testBooleanConversionsLocaleAware(bool $useStringBinder, ?bool $preserveBoolString, $b4Value, $b5Value): void
+    {
+        if ($useStringBinder) {
+            $binder = new StringValueBinder();
+            if (is_bool($preserveBoolString)) {
+                $binder->setBooleanConversion($preserveBoolString);
+            }
+            Cell::setValueBinder($binder);
+        }
+
+        Calculation::getInstance()->setLocale('fr');
+
+        $reader = new Csv();
+        $filename = 'tests/data/Reader/CSV/issue.2232.csv';
+        $spreadsheet = $reader->load($filename);
+        $sheet = $spreadsheet->getActiveSheet();
+        self::assertSame($b4Value, $sheet->getCell('B4')->getValue());
+        self::assertSame($b5Value, $sheet->getCell('B5')->getValue());
+        $spreadsheet->disconnectWorksheets();
+    }
+
+    public function providerIssue2232locale(): array
+    {
+        return [
+            [true, true, 'Faux', 'Vrai'],
+            [true, true, 'Faux', 'Vrai'],
+            [false, false, false, true],
+            [false, false, false, true],
+        ];
+    }
 }

tests/PhpSpreadsheetTests/Reader/Csv/CsvNumberFormatLocaleTest.php

@@ -0,0 +1,145 @@
+<?php
+
+namespace PhpOffice\PhpSpreadsheetTests\Reader\Csv;
+
+use PhpOffice\PhpSpreadsheet\Cell\DataType;
+use PhpOffice\PhpSpreadsheet\Reader\Csv;
+use PHPUnit\Framework\TestCase;
+
+class CsvNumberFormatLocaleTest extends TestCase
+{
+    /**
+     * @var bool
+     */
+    private $localeAdjusted;
+
+    /**
+     * @var false|string
+     */
+    private $currentLocale;
+
+    /**
+     * @var string
+     */
+    protected $filename;
+
+    /**
+     * @var Csv
+     */
+    protected $csvReader;
+
+    protected function setUp(): void
+    {
+        $this->currentLocale = setlocale(LC_ALL, '0');
+
+        if (!setlocale(LC_ALL, 'de_DE.UTF-8', 'deu_deu')) {
+            $this->localeAdjusted = false;
+
+            return;
+        }
+
+        $this->localeAdjusted = true;
+
+        $this->filename = 'tests/data/Reader/CSV/NumberFormatTest.de.csv';
+        $this->csvReader = new Csv();
+    }
+
+    protected function tearDown(): void
+    {
+        if ($this->localeAdjusted && is_string($this->currentLocale)) {
+            setlocale(LC_ALL, $this->currentLocale);
+        }
+    }
+
+    /**
+     * @dataProvider providerNumberFormatNoConversionTest
+     *
+     * @param mixed $expectedValue
+     */
+    public function testNumberFormatNoConversion($expectedValue, string $expectedFormat, string $cellAddress): void
+    {
+        if (!$this->localeAdjusted) {
+            self::markTestSkipped('Unable to set locale for testing.');
+        }
+
+        $spreadsheet = $this->csvReader->load($this->filename);
+        $worksheet = $spreadsheet->getActiveSheet();
+
+        $cell = $worksheet->getCell($cellAddress);
+
+        self::assertSame($expectedValue, $cell->getValue(), 'Expected value check');
+        self::assertSame($expectedFormat, $cell->getFormattedValue(), 'Format mask check');
+    }
+
+    public function providerNumberFormatNoConversionTest(): array
+    {
+        return [
+            [
+                -123,
+                '-123',
+                'A1',
+            ],
+            [
+                '12.345,67',
+                '12.345,67',
+                'C1',
+            ],
+            [
+                '-1.234,567',
+                '-1.234,567',
+                'A3',
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider providerNumberValueConversionTest
+     *
+     * @param mixed $expectedValue
+     */
+    public function testNumberValueConversion($expectedValue, string $cellAddress): void
+    {
+        if (!$this->localeAdjusted) {
+            self::markTestSkipped('Unable to set locale for testing.');
+        }
+
+        $this->csvReader->castFormattedNumberToNumeric(true);
+        $spreadsheet = $this->csvReader->load($this->filename);
+        $worksheet = $spreadsheet->getActiveSheet();
+
+        $cell = $worksheet->getCell($cellAddress);
+
+        self::assertSame(DataType::TYPE_NUMERIC, $cell->getDataType(), 'Datatype check');
+        self::assertSame($expectedValue, $cell->getValue(), 'Expected value check');
+    }
+
+    public function providerNumberValueConversionTest(): array
+    {
+        return [
+            'A1' => [
+                -123,
+                'A1',
+            ],
+            'B1' => [
+                1234,
+                'B1',
+            ],
+            'C1' => [
+                12345.67,
+                'C1',
+            ],
+            'A2' => [
+                123.4567,
+                'A2',
+            ],
+            'B2' => [
+                123.456789012,
+                'B2',
+            ],
+            'A3' => [
+                -1234.567,
+                'A3',
+            ],
+        ];
+    }
+}

tests/PhpSpreadsheetTests/Reader/Csv/CsvNumberFormatTest.php

@@ -0,0 +1,173 @@
+<?php
+
+namespace PhpOffice\PhpSpreadsheetTests\Reader\Csv;
+
+use PhpOffice\PhpSpreadsheet\Cell\DataType;
+use PhpOffice\PhpSpreadsheet\Reader\Csv;
+use PHPUnit\Framework\TestCase;
+
+class CsvNumberFormatTest extends TestCase
+{
+    /**
+     * @var string
+     */
+    protected $filename;
+
+    /**
+     * @var Csv
+     */
+    protected $csvReader;
+
+    protected function setUp(): void
+    {
+        $this->filename = 'tests/data/Reader/CSV/NumberFormatTest.csv';
+        $this->csvReader = new Csv();
+    }
+
+    /**
+     * @dataProvider providerNumberFormatNoConversionTest
+     *
+     * @param mixed $expectedValue
+     */
+    public function testNumberFormatNoConversion($expectedValue, string $expectedFormat, string $cellAddress): void
+    {
+        $spreadsheet = $this->csvReader->load($this->filename);
+        $worksheet = $spreadsheet->getActiveSheet();
+
+        $cell = $worksheet->getCell($cellAddress);
+
+        self::assertSame($expectedValue, $cell->getValue(), 'Expected value check');
+        self::assertSame($expectedFormat, $cell->getFormattedValue(), 'Format mask check');
+    }
+
+    public function providerNumberFormatNoConversionTest(): array
+    {
+        return [
+            [
+                -123,
+                '-123',
+                'A1',
+            ],
+            [
+                '12,345.67',
+                '12,345.67',
+                'C1',
+            ],
+            [
+                '-1,234.567',
+                '-1,234.567',
+                'A3',
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider providerNumberValueConversionTest
+     *
+     * @param mixed $expectedValue
+     */
+    public function testNumberValueConversion($expectedValue, string $cellAddress): void
+    {
+        $this->csvReader->castFormattedNumberToNumeric(true);
+        $spreadsheet = $this->csvReader->load($this->filename);
+        $worksheet = $spreadsheet->getActiveSheet();
+
+        $cell = $worksheet->getCell($cellAddress);
+
+        self::assertSame(DataType::TYPE_NUMERIC, $cell->getDataType(), 'Datatype check');
+        self::assertSame($expectedValue, $cell->getValue(), 'Expected value check');
+    }
+
+    public function providerNumberValueConversionTest(): array
+    {
+        return [
+            'A1' => [
+                -123,
+                'A1',
+            ],
+            'B1' => [
+                1234,
+                'B1',
+            ],
+            'C1' => [
+                12345.67,
+                'C1',
+            ],
+            'A2' => [
+                123.4567,
+                'A2',
+            ],
+            'B2' => [
+                123.456789012,
+                'B2',
+            ],
+            'A3' => [
+                -1234.567,
+                'A3',
+            ],
+            'B3' => [
+                1234.567,
+                'B3',
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider providerNumberFormatConversionTest
+     *
+     * @param mixed $expectedValue
+     */
+    public function testNumberFormatConversion($expectedValue, string $expectedFormat, string $cellAddress): void
+    {
+        $this->csvReader->castFormattedNumberToNumeric(true, true);
+        $spreadsheet = $this->csvReader->load($this->filename);
+        $worksheet = $spreadsheet->getActiveSheet();
+
+        $cell = $worksheet->getCell($cellAddress);
+
+        self::assertSame(DataType::TYPE_NUMERIC, $cell->getDataType(), 'Datatype check');
+        self::assertSame($expectedValue, $cell->getValue(), 'Expected value check');
+        self::assertSame($expectedFormat, $cell->getFormattedValue(), 'Format mask check');
+    }
+
+    public function providerNumberFormatConversionTest(): array
+    {
+        return [
+            'A1' => [
+                -123,
+                '-123',
+                'A1',
+            ],
+            'B1' => [
+                1234,
+                '1,234',
+                'B1',
+            ],
+            'C1' => [
+                12345.67,
+                '12,345.67',
+                'C1',
+            ],
+            'A2' => [
+                123.4567,
+                '123.4567',
+                'A2',
+            ],
+            'B2' => [
+                123.456789012,
+                '123.456789',
+                'B2',
+            ],
+            'A3' => [
+                -1234.567,
+                '-1,234.567',
+                'A3',
+            ],
+            'B3' => [
+                1234.567,
+                '1234.567',
+                'B3',
+            ],
+        ];
+    }
+}

tests/data/Reader/CSV/NumberFormatTest.csv

@@ -0,0 +1,3 @@
+"-123","1,234","12,345.67"
+"123.4567","123.456789012"
+"-1,234.567",1234.567

tests/data/Reader/CSV/NumberFormatTest.de.csv

@@ -0,0 +1,3 @@
+"-123","1.234","12.345,67"
+"123,4567","123,456789012"
+"-1.234,567"

tests/data/Reader/CSV/issue.2232.csv

@@ -1,3 +1,5 @@
 1,2,3
 a,FaLSe,b
 cc,tRUE,cc
+dd,Faux,ee
+ff,Vrai,gg