1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
|
{-# LANGUAGE NoImplicitPrelude
, BangPatterns
, TypeApplications
, MultiWayIf
#-}
{-# OPTIONS_GHC -funbox-strict-fields #-}
module System.OsString.Encoding.Internal where
import qualified System.OsString.Data.ByteString.Short as BS8
import qualified System.OsString.Data.ByteString.Short.Word16 as BS16
import GHC.Base
import GHC.Real
import GHC.Num
-- import GHC.IO
import GHC.IO.Buffer
import GHC.IO.Encoding.Failure
import GHC.IO.Encoding.Types
import Data.Bits
import Control.Exception (SomeException, try, Exception (displayException), evaluate)
import qualified GHC.Foreign as GHC
import Data.Either (Either)
import GHC.IO (unsafePerformIO)
import Control.DeepSeq (force, NFData (rnf))
import Data.Bifunctor (first)
import Data.Data (Typeable)
import GHC.Show (Show (show))
import Numeric (showHex)
import Foreign.C (CStringLen)
import Data.Char (chr)
import Foreign
import GHC.IO.Encoding (getFileSystemEncoding, getLocaleEncoding)
-- -----------------------------------------------------------------------------
-- UCS-2 LE
--
ucs2le :: TextEncoding
ucs2le = mkUcs2le ErrorOnCodingFailure
mkUcs2le :: CodingFailureMode -> TextEncoding
mkUcs2le cfm = TextEncoding { textEncodingName = "UCS-2LE",
mkTextDecoder = ucs2le_DF cfm,
mkTextEncoder = ucs2le_EF cfm }
ucs2le_DF :: CodingFailureMode -> IO (TextDecoder ())
ucs2le_DF cfm =
return (BufferCodec {
encode = ucs2le_decode,
recover = recoverDecode cfm,
close = return (),
getState = return (),
setState = const $ return ()
})
ucs2le_EF :: CodingFailureMode -> IO (TextEncoder ())
ucs2le_EF cfm =
return (BufferCodec {
encode = ucs2le_encode,
recover = recoverEncode cfm,
close = return (),
getState = return (),
setState = const $ return ()
})
ucs2le_decode :: DecodeBuffer
ucs2le_decode
input@Buffer{ bufRaw=iraw, bufL=ir0, bufR=iw, bufSize=_ }
output@Buffer{ bufRaw=oraw, bufL=_, bufR=ow0, bufSize=os }
= let
loop !ir !ow
| ow >= os = done OutputUnderflow ir ow
| ir >= iw = done InputUnderflow ir ow
| ir + 1 == iw = done InputUnderflow ir ow
| otherwise = do
c0 <- readWord8Buf iraw ir
c1 <- readWord8Buf iraw (ir+1)
let x1 = fromIntegral c1 `shiftL` 8 + fromIntegral c0
ow' <- writeCharBuf oraw ow (unsafeChr x1)
loop (ir+2) ow'
-- lambda-lifted, to avoid thunks being built in the inner-loop:
done why !ir !ow = return (why,
if ir == iw then input{ bufL=0, bufR=0 }
else input{ bufL=ir },
output{ bufR=ow })
in
loop ir0 ow0
ucs2le_encode :: EncodeBuffer
ucs2le_encode
input@Buffer{ bufRaw=iraw, bufL=ir0, bufR=iw, bufSize=_ }
output@Buffer{ bufRaw=oraw, bufL=_, bufR=ow0, bufSize=os }
= let
done why !ir !ow = return (why,
if ir == iw then input{ bufL=0, bufR=0 }
else input{ bufL=ir },
output{ bufR=ow })
loop !ir !ow
| ir >= iw = done InputUnderflow ir ow
| os - ow < 2 = done OutputUnderflow ir ow
| otherwise = do
(c,ir') <- readCharBuf iraw ir
case ord c of
x | x < 0x10000 -> do
writeWord8Buf oraw ow (fromIntegral x)
writeWord8Buf oraw (ow+1) (fromIntegral (x `shiftR` 8))
loop ir' (ow+2)
| otherwise -> done InvalidSequence ir ow
in
loop ir0 ow0
-- -----------------------------------------------------------------------------
-- UTF-16b
--
-- | Mimics the base encoding for filesystem operations. This should be total on all inputs (word16 byte arrays).
--
-- Note that this has a subtle difference to 'encodeWithBaseWindows'/'decodeWithBaseWindows': it doesn't care for
-- the @0x0000@ end marker and will as such produce different results. Use @takeWhile (/= '\NUL')@ on the input
-- to recover this behavior.
utf16le_b :: TextEncoding
utf16le_b = mkUTF16le_b ErrorOnCodingFailure
mkUTF16le_b :: CodingFailureMode -> TextEncoding
mkUTF16le_b cfm = TextEncoding { textEncodingName = "UTF-16LE_b",
mkTextDecoder = utf16le_b_DF cfm,
mkTextEncoder = utf16le_b_EF cfm }
utf16le_b_DF :: CodingFailureMode -> IO (TextDecoder ())
utf16le_b_DF cfm =
return (BufferCodec {
encode = utf16le_b_decode,
recover = recoverDecode cfm,
close = return (),
getState = return (),
setState = const $ return ()
})
utf16le_b_EF :: CodingFailureMode -> IO (TextEncoder ())
utf16le_b_EF cfm =
return (BufferCodec {
encode = utf16le_b_encode,
recover = recoverEncode cfm,
close = return (),
getState = return (),
setState = const $ return ()
})
utf16le_b_decode :: DecodeBuffer
utf16le_b_decode
input@Buffer{ bufRaw=iraw, bufL=ir0, bufR=iw, bufSize=_ }
output@Buffer{ bufRaw=oraw, bufL=_, bufR=ow0, bufSize=os }
= let
loop !ir !ow
| ow >= os = done OutputUnderflow ir ow
| ir >= iw = done InputUnderflow ir ow
| ir + 1 == iw = done InputUnderflow ir ow
| otherwise = do
c0 <- readWord8Buf iraw ir
c1 <- readWord8Buf iraw (ir+1)
let x1 = fromIntegral c1 `shiftL` 8 + fromIntegral c0
if | iw - ir >= 4 -> do
c2 <- readWord8Buf iraw (ir+2)
c3 <- readWord8Buf iraw (ir+3)
let x2 = fromIntegral c3 `shiftL` 8 + fromIntegral c2
if | 0xd800 <= x1 && x1 <= 0xdbff
, 0xdc00 <= x2 && x2 <= 0xdfff -> do
ow' <- writeCharBuf oraw ow (unsafeChr ((x1 - 0xd800)*0x400 + (x2 - 0xdc00) + 0x10000))
loop (ir+4) ow'
| otherwise -> do
ow' <- writeCharBuf oraw ow (unsafeChr x1)
loop (ir+2) ow'
| iw - ir >= 2 -> do
ow' <- writeCharBuf oraw ow (unsafeChr x1)
loop (ir+2) ow'
| otherwise -> done InputUnderflow ir ow
-- lambda-lifted, to avoid thunks being built in the inner-loop:
done why !ir !ow = return (why,
if ir == iw then input{ bufL=0, bufR=0 }
else input{ bufL=ir },
output{ bufR=ow })
in
loop ir0 ow0
utf16le_b_encode :: EncodeBuffer
utf16le_b_encode
input@Buffer{ bufRaw=iraw, bufL=ir0, bufR=iw, bufSize=_ }
output@Buffer{ bufRaw=oraw, bufL=_, bufR=ow0, bufSize=os }
= let
done why !ir !ow = return (why,
if ir == iw then input{ bufL=0, bufR=0 }
else input{ bufL=ir },
output{ bufR=ow })
loop !ir !ow
| ir >= iw = done InputUnderflow ir ow
| os - ow < 2 = done OutputUnderflow ir ow
| otherwise = do
(c,ir') <- readCharBuf iraw ir
case ord c of
x | x < 0x10000 -> do
writeWord8Buf oraw ow (fromIntegral x)
writeWord8Buf oraw (ow+1) (fromIntegral (x `shiftR` 8))
loop ir' (ow+2)
| otherwise ->
if os - ow < 4 then done OutputUnderflow ir ow else do
let x' = x - 0x10000
w1 = x' `div` 0x400 + 0xd800
w2 = x' `mod` 0x400 + 0xdc00
writeWord8Buf oraw ow (fromIntegral w1)
writeWord8Buf oraw (ow+1) (fromIntegral (w1 `shiftR` 8))
writeWord8Buf oraw (ow+2) (fromIntegral w2)
writeWord8Buf oraw (ow+3) (fromIntegral (w2 `shiftR` 8))
loop ir' (ow+4)
in
loop ir0 ow0
-- -----------------------------------------------------------------------------
-- Windows encoding (ripped off from base)
--
cWcharsToChars_UCS2 :: [Word16] -> [Char]
cWcharsToChars_UCS2 = map (chr . fromIntegral)
-- On Windows, wchar_t is 16 bits wide and CWString uses the UTF-16 encoding.
-- coding errors generate Chars in the surrogate range
cWcharsToChars :: [Word16] -> [Char]
cWcharsToChars = map chr . fromUTF16 . map fromIntegral
where
fromUTF16 :: [Int] -> [Int]
fromUTF16 (c1:c2:wcs)
| 0xd800 <= c1 && c1 <= 0xdbff && 0xdc00 <= c2 && c2 <= 0xdfff =
((c1 - 0xd800)*0x400 + (c2 - 0xdc00) + 0x10000) : fromUTF16 wcs
fromUTF16 (c:wcs) = c : fromUTF16 wcs
fromUTF16 [] = []
charsToCWchars :: [Char] -> [Word16]
charsToCWchars = foldr (utf16Char . ord) []
where
utf16Char :: Int -> [Word16] -> [Word16]
utf16Char c wcs
| c < 0x10000 = fromIntegral c : wcs
| otherwise = let c' = c - 0x10000 in
fromIntegral (c' `div` 0x400 + 0xd800) :
fromIntegral (c' `mod` 0x400 + 0xdc00) : wcs
-- -----------------------------------------------------------------------------
-- -----------------------------------------------------------------------------
-- FFI
--
withWindowsString :: String -> (Int -> Ptr Word16 -> IO a) -> IO a
withWindowsString = withArrayLen . charsToCWchars
peekWindowsString :: (Ptr Word16, Int) -> IO String
peekWindowsString (cp, l) = do
cs <- peekArray l cp
return (cWcharsToChars cs)
withPosixString :: String -> (CStringLen -> IO a) -> IO a
withPosixString fp f = getFileSystemEncoding >>= \enc -> GHC.withCStringLen enc fp f
withPosixString' :: String -> (CStringLen -> IO a) -> IO a
withPosixString' fp f = getLocaleEncoding >>= \enc -> GHC.withCStringLen enc fp f
peekPosixString :: CStringLen -> IO String
peekPosixString fp = getFileSystemEncoding >>= \enc -> GHC.peekCStringLen enc fp
peekPosixString' :: CStringLen -> IO String
peekPosixString' fp = getLocaleEncoding >>= \enc -> GHC.peekCStringLen enc fp
-- | Decode with the given 'TextEncoding'.
decodeWithTE :: TextEncoding -> BS8.ShortByteString -> Either EncodingException String
decodeWithTE enc ba = unsafePerformIO $ do
r <- try @SomeException $ BS8.useAsCStringLen ba $ \fp -> GHC.peekCStringLen enc fp
evaluate $ force $ first (flip EncodingError Nothing . displayException) r
-- | Encode with the given 'TextEncoding'.
encodeWithTE :: TextEncoding -> String -> Either EncodingException BS8.ShortByteString
encodeWithTE enc str = unsafePerformIO $ do
r <- try @SomeException $ GHC.withCStringLen enc str $ \cstr -> BS8.packCStringLen cstr
evaluate $ force $ first (flip EncodingError Nothing . displayException) r
-- -----------------------------------------------------------------------------
-- Encoders / decoders
--
-- | This mimics the filepath decoder base uses on unix (using PEP-383),
-- with the small distinction that we're not truncating at NUL bytes (because we're not at
-- the outer FFI layer).
decodeWithBasePosix :: BS8.ShortByteString -> IO String
decodeWithBasePosix ba = BS8.useAsCStringLen ba $ \fp -> peekPosixString fp
-- | This mimics the string decoder base uses on unix,
-- with the small distinction that we're not truncating at NUL bytes (because we're not at
-- the outer FFI layer).
decodeWithBasePosix' :: BS8.ShortByteString -> IO String
decodeWithBasePosix' ba = BS8.useAsCStringLen ba $ \fp -> peekPosixString' fp
-- | This mimics the filepath encoder base uses on unix (using PEP-383),
-- with the small distinction that we're not truncating at NUL bytes (because we're not at
-- the outer FFI layer).
encodeWithBasePosix :: String -> IO BS8.ShortByteString
encodeWithBasePosix str = withPosixString str $ \cstr -> BS8.packCStringLen cstr
-- | This mimics the string encoder base uses on unix,
-- with the small distinction that we're not truncating at NUL bytes (because we're not at
-- the outer FFI layer).
encodeWithBasePosix' :: String -> IO BS8.ShortByteString
encodeWithBasePosix' str = withPosixString' str $ \cstr -> BS8.packCStringLen cstr
-- | This mimics the filepath decoder base uses on windows,
-- with the small distinction that we're not truncating at NUL bytes (because we're not at
-- the outer FFI layer).
decodeWithBaseWindows :: BS16.ShortByteString -> IO String
decodeWithBaseWindows ba = BS16.useAsCWStringLen ba $ \fp -> peekWindowsString fp
-- | This mimics the filepath dencoder base uses on windows,
-- with the small distinction that we're not truncating at NUL bytes (because we're not at
-- the outer FFI layer).
encodeWithBaseWindows :: String -> IO BS16.ShortByteString
encodeWithBaseWindows str = withWindowsString str $ \l cstr -> BS16.packCWStringLen (cstr, l)
-- -----------------------------------------------------------------------------
-- Types
--
data EncodingException =
EncodingError String (Maybe Word8)
-- ^ Could not decode a byte sequence because it was invalid under
-- the given encoding, or ran out of input in mid-decode.
deriving (Eq, Typeable)
showEncodingException :: EncodingException -> String
showEncodingException (EncodingError desc (Just w))
= "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc)
showEncodingException (EncodingError desc Nothing)
= "Cannot decode input: " ++ desc
instance Show EncodingException where
show = showEncodingException
instance Exception EncodingException
instance NFData EncodingException where
rnf (EncodingError desc w) = rnf desc `seq` rnf w
-- -----------------------------------------------------------------------------
-- Words
--
wNUL :: Word16
wNUL = 0x00
|