Use ZnUTF8Encoder in Pharo9

Johan Brichau · Johan Brichau · commit 4e314bca4817 · 2020-12-26T10:31:26.000+01:00
diff --git a/repository/Grease-Pharo90-Core.package/GRPharoRandomProvider.class/class/nextInt..st b/repository/Grease-Pharo90-Core.package/GRPharoRandomProvider.class/class/nextInt..st
@@ -1,5 +1,6 @@
 public
 nextInt: anInteger
+
 	"Answer a random integer in the interval [1, anInteger]"
 
-	^ mutex critical: [ generator nextInt: anInteger ]
+	^ mutex critical: [ generator nextInteger: anInteger ]
diff --git a/repository/Grease-Pharo90-Core.package/GRPharoUtf8Codec.class/instance/decode..st b/repository/Grease-Pharo90-Core.package/GRPharoUtf8Codec.class/instance/decode..st
@@ -1,36 +1,3 @@
-decoding
-decode: aString
-	"Convert the given string from UTF-8 using the fast path if converting to Latin-1"
-	| outStream byte1 byte2 byte3 byte4 unicode stream |
-	stream := aString readStream.
-	outStream := WriteStream on: (String new: aString size).
-	[ stream atEnd not ] whileTrue: [
-		byte1 := stream next asInteger.
-		unicode := byte1.
-		(byte1 bitAnd: 16rE0) = 192 ifTrue: [ "two bytes"
-			byte2 := stream next asInteger.
-			(byte2 bitAnd: 16rC0) = 16r80 ifFalse: [ self invalidUtf8 ].
-			unicode := ((byte1 bitAnd: 31) bitShift: 6) + (byte2 bitAnd: 63) ].
-		(byte1 bitAnd: 16rF0) = 224 ifTrue: [ "three bytes"
-			byte2 := stream next asInteger.
-			(byte2 bitAnd: 16rC0) = 16r80 ifFalse: [ self invalidUtf8 ].
-			byte3 := stream next asInteger.
-			(byte3 bitAnd: 16rC0) = 16r80 ifFalse: [ self invalidUtf8 ].
-			unicode := ((byte1 bitAnd: 15) bitShift: 12) + ((byte2 bitAnd: 63) bitShift: 6)
-				+ (byte3 bitAnd: 63) ].
-		(byte1 bitAnd: 16rF8) = 240 ifTrue: [ "four bytes"
-			byte2 := stream next asInteger.
-			(byte2 bitAnd: 16rC0) = 16r80 ifFalse: [ self invalidUtf8 ].
-			byte3 := stream next asInteger.
-			(byte3 bitAnd: 16rC0) = 16r80 ifFalse: [ self invalidUtf8 ].
-			byte4 := stream next asInteger.
-			(byte4 bitAnd: 16rC0) = 16r80 ifFalse: [ self invalidUtf8 ].
-			unicode := ((byte1 bitAnd: 16r7) bitShift: 18) +
-							((byte2 bitAnd: 63) bitShift: 12) + 
-							((byte3 bitAnd: 63) bitShift: 6) +
-							(byte4 bitAnd: 63) ].
-		unicode ifNil: [ self invalidUtf8 ].
-		unicode = 16rFEFF "ignore BOM" ifFalse: [
-			outStream nextPut: (Character codePoint: unicode) ].
-		unicode := nil ].
-	^ outStream contents
+convenience
+decode: aByteArray
+	^ aByteArray utf8Decoded
diff --git a/repository/Grease-Pharo90-Core.package/GRPharoUtf8Codec.class/instance/decoderFor..st b/repository/Grease-Pharo90-Core.package/GRPharoUtf8Codec.class/instance/decoderFor..st
@@ -1,5 +1,5 @@
 conversion
 decoderFor: aStream 
-	^ GRPharoUtf8CodecStream 
+	^ GRPharoConverterCodecStream 
 		on: aStream
-		converter: UTF8TextConverter new
+		converter: ZnCharacterEncoder utf8
diff --git a/repository/Grease-Pharo90-Core.package/GRPharoUtf8Codec.class/instance/encode..st b/repository/Grease-Pharo90-Core.package/GRPharoUtf8Codec.class/instance/encode..st
@@ -0,0 +1,6 @@
+convenience
+encode: aString
+	| writeStream |
+	writeStream := self encoderFor: (ByteArray new: aString size) writeStream.
+	writeStream nextPutAll: aString.
+	^ writeStream contents
diff --git a/repository/Grease-Pharo90-Core.package/GRPharoUtf8Codec.class/instance/encoderFor..st b/repository/Grease-Pharo90-Core.package/GRPharoUtf8Codec.class/instance/encoderFor..st
@@ -1,5 +1,5 @@
 conversion
 encoderFor: aStream 
-	^ GRPharoUtf8CodecStream 
+	^ GRPharoConverterCodecStream 
 		on: aStream
-		converter: UTF8TextConverter new
+		converter: ZnUTF8Encoder new
diff --git a/repository/Grease-Pharo90-Core.package/GRZnUtf8CodecStream.class/README.md b/repository/Grease-Pharo90-Core.package/GRZnUtf8CodecStream.class/README.md
diff --git a/repository/Grease-Pharo90-Core.package/GRZnUtf8CodecStream.class/instance/nextPutAll..st b/repository/Grease-Pharo90-Core.package/GRZnUtf8CodecStream.class/instance/nextPutAll..st
@@ -0,0 +1,5 @@
+streaming
+nextPutAll: aString
+	"Convert the given string from UTF-8 using the fast path if converting to Latin-1"
+	1 to: aString size by: 1 do: [ :index |
+		converter nextPut: (aString at: index) toStream: stream ]
diff --git a/repository/Grease-Pharo90-Core.package/GRZnUtf8CodecStream.class/properties.json b/repository/Grease-Pharo90-Core.package/GRZnUtf8CodecStream.class/properties.json
@@ -0,0 +1,14 @@
+{
+	"commentStamp" : "",
+	"super" : "GRPharoConverterCodecStream",
+	"category" : "Grease-Pharo90-Core",
+	"classinstvars" : [ ],
+	"pools" : [ ],
+	"classvars" : [
+		"Latin1ToUtf8Encodings",
+		"Latin1ToUtf8Map"
+	],
+	"instvars" : [ ],
+	"name" : "GRZnUtf8CodecStream",
+	"type" : "normal"
+}
diff --git a/repository/Grease-Tests-Core.package/GRCountingStreamTest.class/instance/setUp.st b/repository/Grease-Tests-Core.package/GRCountingStreamTest.class/instance/setUp.st
@@ -1,5 +1,5 @@
 running
 setUp
 	| codecStream |
-	codecStream := ((GRCodec forEncoding: 'utf-8') encoderFor: (WriteStream on: String new)).
+	codecStream := ((GRCodec forEncoding: 'utf-8') encoderFor: (WriteStream on: ByteArray new)).
 	countingStream := GRCountingStream on: codecStream
diff --git a/repository/Grease-Tests-Core.package/GRCountingStreamTest.class/instance/testNextPut.st b/repository/Grease-Tests-Core.package/GRCountingStreamTest.class/instance/testNextPut.st
@@ -3,4 +3,4 @@ testNextPut
 	countingStream nextPut: (Character codePoint: 16rE4).
 	self assert: countingStream size = 2.
 	self assert: countingStream count = 1.
-	self assert: countingStream contents = (String with:  (Character codePoint: 16rC3) with:  (Character codePoint: 16rA4))
+	self assert: countingStream contents = (String with: (Character codePoint: 16rC3) with: (Character codePoint: 16rA4)) asByteArray
diff --git a/repository/Grease-Tests-Core.package/GRCountingStreamTest.class/instance/testNextPutAll.st b/repository/Grease-Tests-Core.package/GRCountingStreamTest.class/instance/testNextPutAll.st
@@ -3,4 +3,4 @@ testNextPutAll
 	countingStream nextPutAll: (String with: (Character codePoint: 16rE4)).
 	self assert: countingStream size = 2.
 	self assert: countingStream count = 1.
-	self assert: countingStream contents = (String with:  (Character codePoint: 16rC3) with:  (Character codePoint: 16rA4))
+	self assert: countingStream contents = (String with: (Character codePoint: 16rC3) with: (Character codePoint: 16rA4)) asByteArray
diff --git a/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/asByteArray..st b/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/asByteArray..st
@@ -0,0 +1,5 @@
+private
+asByteArray: aCollectionOfIntegers
+	^ ByteArray streamContents: [ :stream |
+		aCollectionOfIntegers do: [ :each |
+			stream nextPut: each ] ]
diff --git a/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/testCodecUtf8.st b/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/testCodecUtf8.st
@@ -5,7 +5,7 @@ testCodecUtf8
 		codec := GRCodec forEncoding: codecName.
 		self assert: codec name asLowercase = codecName asLowercase.
 		self assert: codec url name asLowercase = codecName asLowercase.
-		self assert: (codec encode: self decodedString) greaseString = self utf8String greaseString.
-		self assert: (codec url encode: self decodedString) greaseString = self utf8String greaseString.
-		self assert: (codec decode: self utf8String) = self decodedString.
-		self assert: (codec url decode: self utf8String) = self decodedString ]
+		self assert: (codec encode: self decodedString) asString = self utf8String.
+		self assert: (codec url encode: self decodedString) asString = self utf8String.
+		self assert: (codec decode: self utf8ByteArray) = self decodedString.
+		self assert: (codec url decode: self utf8ByteArray) = self decodedString ]
diff --git a/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/testCodecUtf8Bom.st b/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/testCodecUtf8Bom.st
@@ -3,6 +3,6 @@ testCodecUtf8Bom
 	#('UTF-8' 'utf-8') do: [ :codecName |
 		| codec bom |
 		codec := GRCodec forEncoding: codecName.
-		bom := self asString: #(239 187 191).
+		bom := self asByteArray: #(239 187 191).
 		self assert: (codec decode: bom , self utf8String) greaseString = self decodedString greaseString.
 		self assert: (codec url decode: bom , self utf8String) greaseString = self decodedString greaseString ]
diff --git a/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/testCodecUtf8BorderLineString.st b/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/testCodecUtf8BorderLineString.st
@@ -3,11 +3,11 @@ testCodecUtf8BorderLineString
 	#('UTF-8' 'utf-8') do: [ :codecName |
 		| codec writeStream |
 		codec := GRCodec forEncoding: codecName.
-		writeStream := codec encoderFor: GRPlatform current readWriteCharacterStream.
+		writeStream := codec encoderFor: GRPlatform current readWriteByteStream.
 		writeStream nextPut: (Character codePoint: 0).
 		writeStream nextPut: (Character codePoint: 255).
 		writeStream nextPut: (Character codePoint: 256).
-		self assert: writeStream contents = (String
+		self assert: writeStream contents asString = (String
 			with: (Character codePoint: 16r00) "character 0" 
 			with: (Character codePoint: 16rC3) with: (Character codePoint: 16rBF) "character 255" 
 			with: (Character codePoint: 16rC4) with: (Character codePoint: 16r80)) "character 256"  ]
diff --git a/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/testCodecUtf8ShortestForm.st b/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/testCodecUtf8ShortestForm.st
@@ -5,7 +5,7 @@ testCodecUtf8ShortestForm
 	#('UTF-8' 'utf-8') do: [ :codecName |
 		| codec abc |
 		codec := GRCodec forEncoding: codecName.
-		abc := self asString: #(193 129 193 130 193 131 ).
+		abc := self asByteArray: #(193 129 193 130 193 131 ).
 		self
 			should: [ self deny: (codec decode: abc) = 'ABC' ]
 			raise: Error ]
diff --git a/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/utf8ByteArray.st b/repository/Grease-Tests-Core.package/GRUtf8CodecTest.class/instance/utf8ByteArray.st
@@ -0,0 +1,3 @@
+accessing
+utf8ByteArray
+	^ self asByteArray: #(195 156 98 195 168 114 115 116 114 195 174 195 177 103 195 169)
diff --git a/repository/Grease-Tests-Pharo-Core.package/GRPharoCodecTest.class/instance/assert.next.startingAt.gives..st b/repository/Grease-Tests-Pharo-Core.package/GRPharoCodecTest.class/instance/assert.next.startingAt.gives..st
@@ -1,7 +1,7 @@
 private
 assert: aString next: anInteger startingAt: startIndex gives: anEncodedString
 	| actual |
-	actual := String streamContents: [ :stream |
+	actual := ByteArray streamContents: [ :stream |
 		((GRCodec forEncoding: 'utf-8') encoderFor: stream)
 			greaseNext: anInteger putAll: aString startingAt: startIndex ].
 	self assert: actual = anEncodedString
diff --git a/repository/Grease-Tests-Pharo-Core.package/GRPharoCodecTest.class/instance/testGreaseNextPutAllStartingAt.st b/repository/Grease-Tests-Pharo-Core.package/GRPharoCodecTest.class/instance/testGreaseNextPutAllStartingAt.st
@@ -2,12 +2,12 @@ tests
 testGreaseNextPutAllStartingAt
 	| umlaut encodedUmlaut |
 	umlaut := String with: (Character codePoint: 228).
-	encodedUmlaut := String with: (Character codePoint: 195) with: (Character codePoint: 164).
-	self assert: 'ab' next: 1 startingAt: 1 gives:  'a'.
-	self assert: 'a', umlaut, 'b' next: 1 startingAt: 1 gives:  'a'.
-	self assert: 'ab', umlaut next: 1 startingAt: 1 gives:  'a'.
-	self assert: 'a', umlaut, 'b' next: 2 startingAt: 1gives:  'a', encodedUmlaut.
-	self assert: 'a', umlaut, 'b' next: 1 startingAt: 2 gives:  encodedUmlaut.
-	self assert: 'a', umlaut, 'b' next: 2 startingAt: 2 gives:  encodedUmlaut, 'b'.
-	self assert: 'a', umlaut, umlaut next: 2 startingAt: 2 gives:  encodedUmlaut, encodedUmlaut.
-	self assert: 'ab', umlaut, 'b', umlaut next: 3 startingAt: 2 gives:  'b', encodedUmlaut, 'b'
+	encodedUmlaut := ByteArray with: (Character codePoint: 195) asInteger with: (Character codePoint: 164) asInteger.
+	self assert: 'ab' next: 1 startingAt: 1 gives:  'a' asByteArray.
+	self assert: 'a', umlaut, 'b' next: 1 startingAt: 1 gives:  'a' asByteArray.
+	self assert: 'ab', umlaut next: 1 startingAt: 1 gives:  'a' asByteArray.
+	self assert: 'a', umlaut, 'b' next: 2 startingAt: 1 gives: 'a' asByteArray, encodedUmlaut.
+	self assert: 'a', umlaut, 'b' next: 1 startingAt: 2 gives: encodedUmlaut.
+	self assert: 'a', umlaut, 'b' next: 2 startingAt: 2 gives: encodedUmlaut, 'b' asByteArray.
+	self assert: 'a', umlaut, umlaut next: 2 startingAt: 2 gives: encodedUmlaut, encodedUmlaut.
+	self assert: 'ab', umlaut, 'b', umlaut next: 3 startingAt: 2 gives: 'b' asByteArray, encodedUmlaut, 'b' asByteArray
diff --git a/repository/Grease-Tests-Pharo-Core.package/GRUtf8CodecTest.extension/instance/expectedFailures.st b/repository/Grease-Tests-Pharo-Core.package/GRUtf8CodecTest.extension/instance/expectedFailures.st
diff --git a/repository/Grease-Tests-Pharo-Core.package/GRUtf8CodecTest.extension/properties.json b/repository/Grease-Tests-Pharo-Core.package/GRUtf8CodecTest.extension/properties.json
