Skip to content

Commit d26b001

Browse files
author
Konrad Michalik
authored
Merge pull request #34 from move-elevator/encoding-validator
feat: enhance EncodingValidator with performance optimizations
2 parents 563cc0e + a6c8413 commit d26b001

6 files changed

Lines changed: 60 additions & 56 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ The following translation validators are available:
8585
| `DuplicateValuesValidator` | This validator checks for duplicate values in translation files. | XLIFF, YAML, JSON, PHP | WARNING |
8686
| `XliffSchemaValidator` | Validates the XML schema of translation files against the XLIFF standard. See available [schemas](https://github.com/symfony/translation/tree/6.4/Resources/schemas). | XLIFF | ERROR |
8787
| `EmptyValuesValidator` | Finds empty or whitespace-only translation values. | XLIFF, YAML, JSON, PHP | WARNING |
88-
| `EncodingValidator` | Validates file encoding, checks for BOM, invisible characters, Unicode normalization issues, and JSON syntax validation. | XLIFF, YAML, JSON, PHP | WARNING |
88+
| `EncodingValidator` | Validates file encoding, checks for BOM, invisible characters and Unicode normalization issues. | XLIFF, YAML, JSON, PHP | WARNING |
8989
| `PlaceholderConsistencyValidator` | Validates placeholder consistency across files. | XLIFF, YAML, JSON, PHP | WARNING |
9090

9191
## 🧑‍💻 Contributing

‎composer-require-checker.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"symbol-whitelist": [
3+
"Normalizer"
4+
]
5+
}

‎composer.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
],
1313
"require": {
1414
"php": "^8.1",
15-
"ext-intl": "*",
1615
"ext-mbstring": "*",
1716
"ext-simplexml": "*",
1817
"composer-plugin-api": "^1.0 || ^2.0",
@@ -36,6 +35,9 @@
3635
"phpunit/phpunit": "^10.2 || ^11.0 || ^12.0",
3736
"roave/security-advisories": "dev-latest"
3837
},
38+
"suggest": {
39+
"ext-intl": "Required for validating translations regarding unicode normalization issues."
40+
},
3941
"autoload": {
4042
"psr-4": {
4143
"MoveElevator\\ComposerTranslationValidator\\": "src"

‎composer.lock‎

Lines changed: 1 addition & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/Validator/EncodingValidator.php‎

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,22 @@ public function processFile(ParserInterface $file): array
3131
return [];
3232
}
3333

34-
// Check UTF-8 encoding
34+
// Early exit for empty files
35+
if ('' === $content) {
36+
return [];
37+
}
38+
39+
// Check UTF-8 encoding first - if invalid, other checks may fail
3540
if (!$this->isValidUtf8($content)) {
3641
$issues['encoding'] = 'File is not valid UTF-8 encoded';
42+
43+
// Skip other checks for invalid UTF-8 content
44+
return $issues;
3745
}
3846

39-
// Check for BOM
40-
if ($this->hasByteOrderMark($content)) {
47+
// Check for BOM (fast byte check)
48+
$hasBom = $this->hasByteOrderMark($content);
49+
if ($hasBom) {
4150
$issues['bom'] = 'File contains UTF-8 Byte Order Mark (BOM)';
4251
}
4352

@@ -50,15 +59,13 @@ public function processFile(ParserInterface $file): array
5059
);
5160
}
5261

53-
// Check Unicode normalization
62+
// Check Unicode normalization (expensive, only if intl available)
5463
if ($this->hasUnicodeNormalizationIssues($content)) {
5564
$issues['unicode_normalization'] = 'File contains non-NFC normalized Unicode characters';
5665
}
5766

58-
// JSON-specific validation for JSON files
59-
if ($file instanceof JsonParser && !$this->isValidJsonStructure($content)) {
60-
$issues['json_syntax'] = 'File contains invalid JSON syntax';
61-
}
67+
// Note: JSON syntax validation is handled by JsonParser constructor
68+
// Invalid JSON files will throw exceptions before reaching this validator
6269

6370
return $issues;
6471
}
@@ -112,19 +119,29 @@ private function findInvisibleCharacters(string $content): array
112119
{
113120
$problematicChars = [];
114121

115-
// Check for various problematic characters
116-
$checks = [
117-
'Zero-width space' => "\u{200B}",
118-
'Zero-width non-joiner' => "\u{200C}",
119-
'Zero-width joiner' => "\u{200D}",
120-
'Word joiner' => "\u{2060}",
121-
'Zero-width no-break space' => "\u{FEFF}",
122-
'Left-to-right mark' => "\u{200E}",
123-
'Right-to-left mark' => "\u{200F}",
124-
'Soft hyphen' => "\u{00AD}",
122+
// Early exit for ASCII-only content (performance optimization)
123+
if (mb_check_encoding($content, 'ASCII')) {
124+
// Only check for control characters in ASCII content
125+
if (preg_match('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/', $content)) {
126+
$problematicChars[] = 'Control characters';
127+
}
128+
129+
return $problematicChars;
130+
}
131+
132+
// Check for problematic Unicode characters individually for better performance
133+
$charMap = [
134+
"\u{200B}" => 'Zero-width space',
135+
"\u{200C}" => 'Zero-width non-joiner',
136+
"\u{200D}" => 'Zero-width joiner',
137+
"\u{2060}" => 'Word joiner',
138+
"\u{FEFF}" => 'Zero-width no-break space',
139+
"\u{200E}" => 'Left-to-right mark',
140+
"\u{200F}" => 'Right-to-left mark',
141+
"\u{00AD}" => 'Soft hyphen',
125142
];
126143

127-
foreach ($checks as $name => $char) {
144+
foreach ($charMap as $char => $name) {
128145
if (str_contains($content, $char)) {
129146
$problematicChars[] = $name;
130147
}
@@ -141,25 +158,11 @@ private function findInvisibleCharacters(string $content): array
141158
private function hasUnicodeNormalizationIssues(string $content): bool
142159
{
143160
if (!class_exists('Normalizer')) {
144-
// If intl extension is not available, skip this check
145161
return false;
146162
}
147163

148-
// Check if content is not in NFC (Canonical Decomposition followed by Canonical Composition)
149164
$normalized = \Normalizer::normalize($content, \Normalizer::FORM_C);
150165

151166
return false !== $normalized && $content !== $normalized;
152167
}
153-
154-
private function isValidJsonStructure(string $content): bool
155-
{
156-
// Remove BOM if present for JSON validation
157-
$cleanContent = $this->hasByteOrderMark($content)
158-
? substr($content, 3)
159-
: $content;
160-
161-
json_decode($cleanContent);
162-
163-
return JSON_ERROR_NONE === json_last_error();
164-
}
165168
}

‎tests/src/Validator/EncodingValidatorTest.php‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -132,23 +132,19 @@ public function testFileWithUnicodeNormalizationIssues(): void
132132
$this->assertStringContainsString('non-NFC normalized', $issues['unicode_normalization']);
133133
}
134134

135-
public function testInvalidJsonSyntax(): void
135+
public function testJsonFilesAreSupported(): void
136136
{
137-
$filePath = $this->testFilesPath.'/invalid-json.json';
138-
file_put_contents($filePath, '{"key": "value",}'); // Trailing comma
137+
$filePath = $this->testFilesPath.'/valid.json';
138+
file_put_contents($filePath, '{"key": "value"}');
139139

140-
// We need to test the validator directly without parsing
141-
// Since the parser would fail before we can validate
142-
$mockParser = $this->createMock(JsonParser::class);
143-
$mockParser->method('getFilePath')->willReturn($filePath);
144-
145-
$issues = $this->validator->processFile($mockParser);
140+
$parser = new JsonParser($filePath);
141+
$issues = $this->validator->processFile($parser);
146142

147-
$this->assertArrayHasKey('json_syntax', $issues);
148-
$this->assertStringContainsString('invalid JSON syntax', $issues['json_syntax']);
143+
// Should not have any encoding issues for valid JSON
144+
$this->assertEmpty($issues);
149145
}
150146

151-
public function testValidJsonWithBomIsHandled(): void
147+
public function testJsonWithBomIsDetected(): void
152148
{
153149
$filePath = $this->testFilesPath.'/valid-json-with-bom.json';
154150
file_put_contents($filePath, "\xEF\xBB\xBF{\"key\": \"value\"}"); // Valid JSON with BOM
@@ -159,21 +155,20 @@ public function testValidJsonWithBomIsHandled(): void
159155

160156
$issues = $this->validator->processFile($mockParser);
161157

162-
// Should have BOM issue but not JSON syntax issue
158+
// Should detect BOM issue
163159
$this->assertArrayHasKey('bom', $issues);
164-
$this->assertArrayNotHasKey('json_syntax', $issues);
165160
}
166161

167-
public function testNonJsonFileDoesNotCheckJsonSyntax(): void
162+
public function testPhpFilesAreSupported(): void
168163
{
169-
$filePath = $this->testFilesPath.'/invalid-json.php';
170-
file_put_contents($filePath, '<?php return ["key" => "value",]; // Trailing comma OK in PHP');
164+
$filePath = $this->testFilesPath.'/valid.php';
165+
file_put_contents($filePath, '<?php return ["key" => "value"]; // Valid PHP');
171166

172167
$parser = new PhpParser($filePath);
173168
$issues = $this->validator->processFile($parser);
174169

175-
// Should not check JSON syntax for PHP files
176-
$this->assertArrayNotHasKey('json_syntax', $issues);
170+
// Should not have any encoding issues for valid PHP
171+
$this->assertEmpty($issues);
177172
}
178173

179174
public function testFileReadError(): void

0 commit comments

Comments
 (0)