From 1c5373dc161947da53ae2ccb711b8c6e46320218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lester=20Carballo=20P=C3=A9rez?= Date: Mon, 12 Feb 2024 18:20:16 -0600 Subject: [PATCH 1/8] Calculation of the necessary cutting and size of the texts according to their character encoding --- src/Buffer.php | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Buffer.php b/src/Buffer.php index bd3b581..ef1e1a3 100644 --- a/src/Buffer.php +++ b/src/Buffer.php @@ -213,7 +213,22 @@ public function read($length = null) } /** - * @param $data + * @param string $data + * @param int $maxLength + * + * @return int + */ + public function lengthBytes($data, $maxLength) + { + if (strtolower($this->streamCharset) != strtolower($this->systemCharset)) { + $data = mb_convert_encoding($data, $this->streamCharset, $this->systemCharset); + } + $data = mb_strcut($data, 0, $maxLength, $this->streamCharset); + return \strlen($data); + } + + /** + * @param string $data * @param int|string $length * @param null $charset * @@ -226,7 +241,9 @@ public function writeString($data, $length = '*', $charset = null) if (isset($data) && (strtolower($charsetFrom) != strtolower($charsetTo))) { $data = mb_convert_encoding($data, $charsetTo, $charsetFrom); } - //file_put_contents("/var/encuestas/test.txt", "To: " . $charsetTo . " FROM:" . $charsetFrom . "\n", FILE_APPEND | LOCK_EX); + if (isset($length) && ($length != '*')) { + $data = mb_strcut($data, 0, $length, $this->charset); + } return $this->write(pack('A' . $length, $data)); } From ea51444d17f115104475b089b6fc415dafb0ff8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lester=20Carballo=20P=C3=A9rez?= Date: Mon, 12 Feb 2024 18:30:57 -0600 Subject: [PATCH 2/8] Delegate the calculation of the required bytes for the ValueSet label, to the buffer --- src/Sav/Record/ValueLabel.php | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/Sav/Record/ValueLabel.php b/src/Sav/Record/ValueLabel.php index 8ef3ae0..900c539 100644 --- a/src/Sav/Record/ValueLabel.php +++ b/src/Sav/Record/ValueLabel.php @@ -97,22 +97,16 @@ public function write(Buffer $buffer) $buffer->writeInt(self::TYPE); $buffer->writeInt(\count($this->labels)); foreach ($this->labels as $item) { - $labelLength = min(mb_strlen($item['label']), self::LABEL_MAX_LENGTH); - $label = mb_substr($item['label'], 0, $labelLength); - $labelLengthBytes = mb_strlen($label, '8bit'); - while ($labelLengthBytes > 255) { - // Strip one char, can be multiple bytes - $label = mb_substr($label, 0, -1); - $labelLengthBytes = mb_strlen($label, '8bit'); - } - + $labelLengthBytes = $buffer->lengthBytes($item['label'], self::LABEL_MAX_LENGTH); + $labelLengthBytesRound = Utils::roundUp($labelLengthBytes + 1, 8) - 1; + if ($convertToDouble) { $item['value'] = Utils::stringToDouble($item['value']); } $buffer->writeDouble($item['value']); $buffer->write(\chr($labelLengthBytes)); - $buffer->writeString($label, Utils::roundUp($labelLengthBytes + 1, 8) - 1); + $buffer->writeString($item['label'], $labelLengthBytesRound); } // Value label variable record. From 15f26cfce36cb7dd1f7e355cb2890c838d180160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lester=20Carballo=20P=C3=A9rez?= Date: Mon, 12 Feb 2024 19:38:54 -0600 Subject: [PATCH 3/8] Delegate the calculation of the required bytes for the variable label, to the buffer --- src/Sav/Record/Variable.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/Sav/Record/Variable.php b/src/Sav/Record/Variable.php index 3ae6eb8..cab6e78 100644 --- a/src/Sav/Record/Variable.php +++ b/src/Sav/Record/Variable.php @@ -141,17 +141,10 @@ public function write(Buffer $buffer) $buffer->writeString($this->name, 8); if ($hasLabel) { - // Maxlength is 255 bytes, since we write utf8 a char can be multiple bytes - $labelLength = min(mb_strlen($this->label), 255); - $label = mb_substr($this->label, 0, $labelLength); - $labelLengthBytes = mb_strlen($label, '8bit'); - while ($labelLengthBytes > 255) { - // Strip one char, can be multiple bytes - $label = mb_substr($label, 0, -1); - $labelLengthBytes = mb_strlen($label, '8bit'); - } + $labelLengthBytes = $buffer->lengthBytes($this->label, self::REAL_VLS_CHUNK); + $labelLengthBytesRound = Utils::roundUp($labelLengthBytes, 4); $buffer->writeInt($labelLengthBytes); - $buffer->writeString($label, Utils::roundUp($labelLengthBytes, 4)); + $buffer->writeString($this->label, $labelLengthBytesRound); } // TODO: test @@ -222,6 +215,6 @@ public function getSegmentName($seg = 0) return mb_strtoupper($this->name); } $sufix = str_pad($str, 2, "_", STR_PAD_LEFT); - return mb_strtoupper(mb_substr($this->name.$sufix, 0, 8)); + return mb_strtoupper(mb_strcut($this->name.$sufix, 0, 8)); } } From f9816454677b4e88b72f2035f39de286bc877c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lester=20Carballo=20P=C3=A9rez?= Date: Mon, 12 Feb 2024 20:09:58 -0600 Subject: [PATCH 4/8] Delegate the calculation of the required bytes for the LongStringValueLabels label, to the buffer --- src/Buffer.php | 6 ++++-- src/Sav/Record/Info/LongStringValueLabels.php | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Buffer.php b/src/Buffer.php index ef1e1a3..4deb7b6 100644 --- a/src/Buffer.php +++ b/src/Buffer.php @@ -218,12 +218,14 @@ public function read($length = null) * * @return int */ - public function lengthBytes($data, $maxLength) + public function lengthBytes($data, $maxLength = null) { if (strtolower($this->streamCharset) != strtolower($this->systemCharset)) { $data = mb_convert_encoding($data, $this->streamCharset, $this->systemCharset); } - $data = mb_strcut($data, 0, $maxLength, $this->streamCharset); + if (isset($maxLength)) { + $data = mb_strcut($data, 0, $maxLength, $this->streamCharset); + } return \strlen($data); } diff --git a/src/Sav/Record/Info/LongStringValueLabels.php b/src/Sav/Record/Info/LongStringValueLabels.php index e804588..5fcb696 100644 --- a/src/Sav/Record/Info/LongStringValueLabels.php +++ b/src/Sav/Record/Info/LongStringValueLabels.php @@ -45,15 +45,17 @@ public function write(Buffer $buffer) throw new \InvalidArgumentException('values required'); } $width = (int) $data['width']; - $localBuffer->writeInt(mb_strlen($varName)); - $localBuffer->writeString($varName, mb_strlen($varName)); + $varLengthBytes = $buffer->lengthBytes($varName); + $localBuffer->writeInt($varLengthBytes); + $localBuffer->writeString($varName, $varLengthBytes); $localBuffer->writeInt($width); $localBuffer->writeInt(Utils::is_countable($data['values']) ? \count($data['values']) : 0); foreach ($data['values'] as $value => $label) { $localBuffer->writeInt($width); $localBuffer->writeString($value, $width); - $localBuffer->writeInt(mb_strlen($label)); - $localBuffer->writeString($label, mb_strlen($label)); + $labelLengthBytes = $buffer->lengthBytes($label); + $localBuffer->writeInt($labelLengthBytes); + $localBuffer->writeString($label, $labelLengthBytes); } } From bdfa024596158e4c0442d940431400cd0c897bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lester=20Carballo=20P=C3=A9rez?= Date: Mon, 12 Feb 2024 20:16:20 -0600 Subject: [PATCH 5/8] Estimate the size in bytes in the data of the VariableAttributes --- src/Sav/Record/Info/VariableAttributes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sav/Record/Info/VariableAttributes.php b/src/Sav/Record/Info/VariableAttributes.php index 46a90b2..5a1e088 100644 --- a/src/Sav/Record/Info/VariableAttributes.php +++ b/src/Sav/Record/Info/VariableAttributes.php @@ -44,7 +44,7 @@ public function write(Buffer $buffer) if ($lines !== []) { $data = implode('/', $lines); - $this->dataCount = mb_strlen($data); + $this->dataCount = \strlen($data); parent::write($buffer); $buffer->writeString($data); } From ad7daac3b7842b97decc1aceb239cc34360f04f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lester=20Carballo=20P=C3=A9rez?= Date: Mon, 12 Feb 2024 22:01:18 -0600 Subject: [PATCH 6/8] Fixing the introduced previous errors --- src/Sav/Record/Variable.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Sav/Record/Variable.php b/src/Sav/Record/Variable.php index cab6e78..32bec79 100644 --- a/src/Sav/Record/Variable.php +++ b/src/Sav/Record/Variable.php @@ -173,8 +173,7 @@ public function write(Buffer $buffer) $buffer->writeInt(0); // No missing values $buffer->writeInt($format); // Print format $buffer->writeInt($format); // Write format - $buffer->writeString($this->getSegmentName($i - 1), 8); - + $buffer->writeString($this->getSegmentName($buffer, $i - 1), 8); $this->writeBlank($buffer, $segmentWidth); } } @@ -204,7 +203,7 @@ public function writeBlank(Buffer $buffer, $width) * * @return string */ - public function getSegmentName($seg = 0) + public function getSegmentName($buffer, $seg = 0) { // TODO: refactory $str = "a"; From 61234298f0b829516dc9190806aac91a967fde78 Mon Sep 17 00:00:00 2001 From: lestcape Date: Mon, 28 Apr 2025 12:04:16 -0600 Subject: [PATCH 7/8] Fix buffer chartcode name --- src/Buffer.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Buffer.php b/src/Buffer.php index 4deb7b6..3751ebe 100644 --- a/src/Buffer.php +++ b/src/Buffer.php @@ -173,7 +173,7 @@ public function readString($length, $round = 0, $charset = null) $charsetFrom = isset($this->charset) ? $this->charset : mb_internal_encoding(); $charsetTo = isset($charset) ? $charset : mb_internal_encoding(); if (isset($str) && (strtolower($charsetFrom) != strtolower($charsetTo))) { - $str = mb_convert_encoding($str, $charsetTo, $this->charset); + $str = mb_convert_encoding($str, $charsetTo, $charsetFrom); } return $str; @@ -218,13 +218,15 @@ public function read($length = null) * * @return int */ - public function lengthBytes($data, $maxLength = null) + public function lengthBytes($data, $maxLength = null, $charset = null) { - if (strtolower($this->streamCharset) != strtolower($this->systemCharset)) { - $data = mb_convert_encoding($data, $this->streamCharset, $this->systemCharset); + $charsetTo = isset($this->charset) ? $this->charset : mb_internal_encoding(); + $charsetFrom = isset($charset) ? $charset : mb_internal_encoding(); + if (isset($data) && (strtolower($charsetTo) != strtolower($charsetFrom))) { + $data = mb_convert_encoding($data, $charsetTo, $charsetFrom); } - if (isset($maxLength)) { - $data = mb_strcut($data, 0, $maxLength, $this->streamCharset); + if (isset($data) && isset($maxLength)) { + $data = mb_strcut($data, 0, $maxLength, $charsetTo); } return \strlen($data); } @@ -244,7 +246,7 @@ public function writeString($data, $length = '*', $charset = null) $data = mb_convert_encoding($data, $charsetTo, $charsetFrom); } if (isset($length) && ($length != '*')) { - $data = mb_strcut($data, 0, $length, $this->charset); + $data = mb_strcut($data, 0, $length, $charsetTo); } return $this->write(pack('A' . $length, $data)); } From d7f3e4d466dea00bfdbc34dfb3f2fdc046facd63 Mon Sep 17 00:00:00 2001 From: lestcape Date: Fri, 9 May 2025 14:57:49 -0600 Subject: [PATCH 8/8] Revert the charset usage for getSegmentName as we really are used the internal encoding as the source for the encoding --- src/Sav/Record/Variable.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Sav/Record/Variable.php b/src/Sav/Record/Variable.php index 32bec79..cd8f7ee 100644 --- a/src/Sav/Record/Variable.php +++ b/src/Sav/Record/Variable.php @@ -173,7 +173,7 @@ public function write(Buffer $buffer) $buffer->writeInt(0); // No missing values $buffer->writeInt($format); // Print format $buffer->writeInt($format); // Write format - $buffer->writeString($this->getSegmentName($buffer, $i - 1), 8); + $buffer->writeString($this->getSegmentName($i - 1), 8); $this->writeBlank($buffer, $segmentWidth); } } @@ -203,7 +203,7 @@ public function writeBlank(Buffer $buffer, $width) * * @return string */ - public function getSegmentName($buffer, $seg = 0) + public function getSegmentName($seg = 0) { // TODO: refactory $str = "a";